diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b051c42..426977b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,27 +4,33 @@ ### New +#### Telemetry [GH-35] + +- **Tracing** [GH-230] aka distributed tracing, provides insight into the full lifecycles, aka traces, of requests to the system, allowing you to pinpoint failures and performance issues. + + - Add [Jaeger](https://opencensus.io/exporters/supported-exporters/go/jaeger/) support. [GH-230] + +- **Metrics** provide quantitative information about processes running inside the system, including counters, gauges, and histograms. + + - Add informational metrics. [GH-227] + - GRPC Metrics Implementation. [GH-218] + + - Additional GRPC server metrics and request sizes + - Improved GRPC metrics implementation internals + - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status` + + - HTTP Metrics Implementation. [GH-220] + + - Support HTTP request sizes on client and server side of proxy + - Improved HTTP metrics implementation internals + - The HTTP method label is now `http_method`, and HTTP status label is now `http_status` + ### Changed -- GRPC Metrics Implementation [GH-218] - - - Additional GRPC server metrics and request sizes - - Improved GRPC metrics implementation internals - - The GRPC method label is now 'grpc_method' and GRPC status is now `grpc_client_status` and `grpc_server_status` - - GRPC version upgraded to v1.22 [GH-219] - -- HTTP Metrics Implementation [GH-220] - - - Support HTTP request sizes on client and server side of proxy - - Improved HTTP metrics implementation internals - - The HTTP method label is now `http_method`, and HTTP status label is now `http_status` - - Add support for large cookie sessions by chunking. [GH-211] - - Prefer [curve](https://wiki.mozilla.org/Security/Server_Side_TLS) X25519 to P256 for TLS connections. [GH-233] - -- Add informational metrics. [GH-227] +- Pomerium and its services will gracefully shutdown on [interrupt signal](http://man7.org/linux/man-pages/man7/signal.7.html). [GH-230] - [Google](https://developers.google.com/identity/protocols/OpenIDConnect) now prompts the user to select a user account (by adding `select_account` to the sign in url). This allows a user who has multiple accounts at the authorization server to select amongst the multiple accounts that they may have current sessions for. ## v0.1.0 diff --git a/authenticate/grpc.go b/authenticate/grpc.go index 232dabad0..c0e2e2801 100644 --- a/authenticate/grpc.go +++ b/authenticate/grpc.go @@ -6,11 +6,14 @@ import ( "fmt" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" pb "github.com/pomerium/pomerium/proto/authenticate" ) // Authenticate takes an encrypted code, and returns the authentication result. func (p *Authenticate) Authenticate(ctx context.Context, in *pb.AuthenticateRequest) (*pb.Session, error) { + _, span := trace.StartSpan(ctx, "authenticate.grpc.Validate") + defer span.End() session, err := sessions.UnmarshalSession(in.Code, p.cipher) if err != nil { return nil, fmt.Errorf("authenticate/grpc: authenticate %v", err) @@ -25,6 +28,9 @@ func (p *Authenticate) Authenticate(ctx context.Context, in *pb.AuthenticateRequ // Validate locally validates a JWT id_token; does NOT do nonce or revokation validation. // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation func (p *Authenticate) Validate(ctx context.Context, in *pb.ValidateRequest) (*pb.ValidateReply, error) { + ctx, span := trace.StartSpan(ctx, "authenticate.grpc.Validate") + defer span.End() + isValid, err := p.provider.Validate(ctx, in.IdToken) if err != nil { return &pb.ValidateReply{IsValid: false}, fmt.Errorf("authenticate/grpc: validate %v", err) @@ -35,10 +41,8 @@ func (p *Authenticate) Validate(ctx context.Context, in *pb.ValidateRequest) (*p // Refresh renews a user's session checks if the session has been revoked using an access token // without reprompting the user. func (p *Authenticate) Refresh(ctx context.Context, in *pb.Session) (*pb.Session, error) { - // todo(bdd): add request id from incoming context - // md, _ := metadata.FromIncomingContext(ctx) - // sublogger := log.With().Str("req_id", md.Get("req_id")[0]).WithContext(ctx) - // sublogger.Info().Msg("tracing sucks!") + ctx, span := trace.StartSpan(ctx, "authenticate.grpc.Refresh") + defer span.End() if in == nil { return nil, fmt.Errorf("authenticate/grpc: session cannot be nil") } diff --git a/authorize/authorize.go b/authorize/authorize.go index bc5fdb6f2..e259dc4dc 100644 --- a/authorize/authorize.go +++ b/authorize/authorize.go @@ -6,7 +6,7 @@ import ( "github.com/pomerium/pomerium/internal/config" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/metrics" + "github.com/pomerium/pomerium/internal/telemetry/metrics" ) // ValidateOptions checks to see if configuration values are valid for the diff --git a/authorize/gprc.go b/authorize/gprc.go index f66bc9df6..e878bf7cf 100644 --- a/authorize/gprc.go +++ b/authorize/gprc.go @@ -4,12 +4,16 @@ package authorize // import "github.com/pomerium/pomerium/authorize" import ( "context" + "github.com/pomerium/pomerium/internal/telemetry/trace" pb "github.com/pomerium/pomerium/proto/authorize" ) // Authorize validates the user identity, device, and context of a request for // a given route. Currently only checks identity. func (a *Authorize) Authorize(ctx context.Context, in *pb.Identity) (*pb.AuthorizeReply, error) { + _, span := trace.StartSpan(ctx, "authorize.grpc.Authorize") + defer span.End() + ok := a.ValidIdentity(in.Route, &Identity{ User: in.User, @@ -23,6 +27,8 @@ func (a *Authorize) Authorize(ctx context.Context, in *pb.Identity) (*pb.Authori // IsAdmin validates the user is an administrative user. func (a *Authorize) IsAdmin(ctx context.Context, in *pb.Identity) (*pb.IsAdminReply, error) { + _, span := trace.StartSpan(ctx, "authorize.grpc.IsAdmin") + defer span.End() ok := a.identityAccess.IsAdmin( &Identity{ Email: in.Email, diff --git a/cmd/pomerium/main.go b/cmd/pomerium/main.go index 4891304fe..10822b975 100644 --- a/cmd/pomerium/main.go +++ b/cmd/pomerium/main.go @@ -1,12 +1,10 @@ package main // import "github.com/pomerium/pomerium/cmd/pomerium" import ( - "errors" "flag" "fmt" "net/http" "os" - "strconv" "time" "github.com/fsnotify/fsnotify" @@ -18,8 +16,9 @@ import ( "github.com/pomerium/pomerium/internal/config" "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/metrics" "github.com/pomerium/pomerium/internal/middleware" + "github.com/pomerium/pomerium/internal/telemetry/metrics" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/urlutil" "github.com/pomerium/pomerium/internal/version" pbAuthenticate "github.com/pomerium/pomerium/proto/authenticate" @@ -36,17 +35,18 @@ func main() { fmt.Println(version.FullVersion()) os.Exit(0) } - opt, err := parseOptions(*configFile) + opt, err := config.ParseOptions(*configFile) if err != nil { log.Fatal().Err(err).Msg("cmd/pomerium: options") } log.Info().Str("version", version.FullVersion()).Msg("cmd/pomerium") - grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey) - grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest), grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))} - grpcServer := grpc.NewServer(grpcOpts...) + + setupMetrics(opt) + setupTracing(opt) + setupHTTPRedirectServer(opt) mux := http.NewServeMux() - + grpcServer := setupGRPCServer(opt) _, err = newAuthenticateService(*opt, mux, grpcServer) if err != nil { log.Fatal().Err(err).Msg("cmd/pomerium: authenticate") @@ -61,65 +61,23 @@ func main() { if err != nil { log.Fatal().Err(err).Msg("cmd/pomerium: proxy") } + defer proxy.AuthenticateClient.Close() + defer proxy.AuthorizeClient.Close() + go viper.WatchConfig() viper.OnConfigChange(func(e fsnotify.Event) { - log.Info(). - Str("file", e.Name). - Msg("cmd/pomerium: configuration file changed") - - opt = handleConfigUpdate(opt, []config.OptionsUpdater{authz, proxy}) + log.Info().Str("file", e.Name).Msg("cmd/pomerium: config file changed") + opt = config.HandleConfigUpdate(*configFile, opt, []config.OptionsUpdater{authz, proxy}) }) - // defer statements ignored anyway : https://stackoverflow.com/a/17888654 - // defer proxyService.AuthenticateClient.Close() - // defer proxyService.AuthorizeClient.Close() - httpOpts := &httputil.Options{ - Addr: opt.Addr, - Cert: opt.Cert, - Key: opt.Key, - CertFile: opt.CertFile, - KeyFile: opt.KeyFile, - ReadTimeout: opt.ReadTimeout, - WriteTimeout: opt.WriteTimeout, - ReadHeaderTimeout: opt.ReadHeaderTimeout, - IdleTimeout: opt.IdleTimeout, + srv, err := httputil.NewTLSServer(configToServerOptions(opt), mainHandler(opt, mux), grpcServer) + if err != nil { + log.Fatal().Err(err).Msg("cmd/pomerium: couldn't start pomerium") } + httputil.Shutdown(srv) - if opt.MetricsAddr != "" { - go newPromListener(opt.MetricsAddr) - metrics.SetBuildInfo(opt.Services) - } - - if srv, err := startRedirectServer(opt.HTTPRedirectAddr); err != nil { - log.Debug().Str("cause", err.Error()).Msg("cmd/pomerium: http redirect server not started") - } else { - defer srv.Close() - } - - if err := httputil.ListenAndServeTLS(httpOpts, wrapMiddleware(opt, mux), grpcServer); err != nil { - log.Fatal().Err(err).Msg("cmd/pomerium: https server") - } -} - -// startRedirectServer starts a http server that redirect HTTP to HTTPS traffic -func startRedirectServer(addr string) (*http.Server, error) { - if addr == "" { - return nil, errors.New("no http redirect addr provided") - } - srv := &http.Server{ - Addr: addr, - ReadTimeout: 5 * time.Second, - WriteTimeout: 5 * time.Second, - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Connection", "close") - url := fmt.Sprintf("https://%s%s", urlutil.StripPort(r.Host), r.URL.String()) - http.Redirect(w, r, url, http.StatusMovedPermanently) - }), - } - log.Info().Str("Addr", addr).Msg("cmd/pomerium: http redirect server started") - go func() { log.Error().Err(srv.ListenAndServe()).Msg("cmd/pomerium: http server closed") }() - return srv, nil + os.Exit(0) } func newAuthenticateService(opt config.Options, mux *http.ServeMux, rpc *grpc.Server) (*authenticate.Authenticate, error) { @@ -159,21 +117,9 @@ func newProxyService(opt config.Options, mux *http.ServeMux) (*proxy.Proxy, erro return service, nil } -func newPromListener(addr string) { - metrics.RegisterView(metrics.HTTPClientViews) - metrics.RegisterView(metrics.HTTPServerViews) - metrics.RegisterView(metrics.GRPCClientViews) - metrics.RegisterView(metrics.GRPCServerViews) - metrics.RegisterInfoMetrics() - metrics.RegisterView(metrics.InfoViews) - - log.Info().Str("MetricsAddr", addr).Msg("cmd/pomerium: starting prometheus endpoint") - log.Error().Err(metrics.NewPromHTTPListener(addr)).Str("MetricsAddr", addr).Msg("cmd/pomerium: could not start metrics exporter") -} - -func wrapMiddleware(o *config.Options, mux http.Handler) http.Handler { +func mainHandler(o *config.Options, mux http.Handler) http.Handler { c := middleware.NewChain() - c = c.Append(metrics.HTTPMetricsHandler("proxy")) + c = c.Append(metrics.HTTPMetricsHandler(o.Services)) c = c.Append(log.NewHandler(log.Logger)) c = c.Append(log.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) { log.FromRequest(r).Debug(). @@ -199,60 +145,61 @@ func wrapMiddleware(o *config.Options, mux http.Handler) http.Handler { return c.Then(mux) } -func parseOptions(configFile string) (*config.Options, error) { - o, err := config.OptionsFromViper(configFile) - if err != nil { - return nil, err +func configToServerOptions(opt *config.Options) *httputil.ServerOptions { + return &httputil.ServerOptions{ + Addr: opt.Addr, + Cert: opt.Cert, + Key: opt.Key, + CertFile: opt.CertFile, + KeyFile: opt.KeyFile, + ReadTimeout: opt.ReadTimeout, + WriteTimeout: opt.WriteTimeout, + ReadHeaderTimeout: opt.ReadHeaderTimeout, + IdleTimeout: opt.IdleTimeout, } - if o.Debug { - log.SetDebugMode() - } - if o.LogLevel != "" { - log.SetLevel(o.LogLevel) - } - metrics.AddPolicyCountCallback(o.Services, func() int64 { - return int64(len(o.Policies)) - }) - - checksumDec, err := strconv.ParseUint(o.Checksum(), 16, 64) - if err != nil { - log.Warn().Err(err).Msg("Could not parse config checksum into decimal") - } - metrics.SetConfigChecksum(o.Services, checksumDec) - - return o, nil } -func handleConfigUpdate(opt *config.Options, services []config.OptionsUpdater) *config.Options { - newOpt, err := parseOptions(*configFile) - if err != nil { - log.Error().Err(err).Msg("cmd/pomerium: could not reload configuration") - metrics.SetConfigInfo(opt.Services, false, "") - return opt - } - optChecksum := opt.Checksum() - newOptChecksum := newOpt.Checksum() - - log.Debug(). - Str("old-checksum", optChecksum). - Str("new-checksum", newOptChecksum). - Msg("cmd/pomerium: configuration file changed") - - if newOptChecksum == optChecksum { - log.Debug().Msg("cmd/pomerium: loaded configuration has not changed") - return opt - } - - log.Info().Str("checksum", newOptChecksum).Msg("cmd/pomerium: checksum changed") - for _, service := range services { - if err := service.UpdateOptions(*newOpt); err != nil { - log.Error().Err(err).Msg("cmd/pomerium: could not update options") - metrics.SetConfigInfo(opt.Services, false, "") +func setupMetrics(opt *config.Options) { + if opt.MetricsAddr != "" { + if handler, err := metrics.PrometheusHandler(); err != nil { + log.Error().Err(err).Msg("cmd/pomerium: couldn't start metrics server") + } else { + serverOpts := &httputil.ServerOptions{Addr: opt.MetricsAddr} + srv := httputil.NewHTTPServer(serverOpts, handler) + go httputil.Shutdown(srv) } } - metrics.AddPolicyCountCallback(newOpt.Services, func() int64 { - return int64(len(newOpt.Policies)) - }) - metrics.SetConfigInfo(newOpt.Services, true, newOptChecksum) - return newOpt +} + +func setupTracing(opt *config.Options) { + if opt.TracingProvider != "" { + tracingOpts := &trace.TracingOptions{ + Provider: opt.TracingProvider, + Service: opt.Services, + Debug: opt.TracingDebug, + JaegerAgentEndpoint: opt.TracingJaegerAgentEndpoint, + JaegerCollectorEndpoint: opt.TracingJaegerCollectorEndpoint, + } + if err := trace.RegisterTracing(tracingOpts); err != nil { + log.Error().Err(err).Msg("cmd/pomerium: couldn't register tracing") + } else { + log.Info().Interface("options", tracingOpts).Msg("cmd/pomerium: metrics configured") + } + } +} + +func setupHTTPRedirectServer(opt *config.Options) { + if opt.HTTPRedirectAddr != "" { + serverOpts := httputil.ServerOptions{Addr: opt.HTTPRedirectAddr} + srv := httputil.NewHTTPServer(&serverOpts, httputil.RedirectHandler()) + go httputil.Shutdown(srv) + } +} + +func setupGRPCServer(opt *config.Options) *grpc.Server { + grpcAuth := middleware.NewSharedSecretCred(opt.SharedKey) + grpcOpts := []grpc.ServerOption{ + grpc.UnaryInterceptor(grpcAuth.ValidateRequest), + grpc.StatsHandler(metrics.NewGRPCServerStatsHandler(opt.Services))} + return grpc.NewServer(grpcOpts...) } diff --git a/cmd/pomerium/main_test.go b/cmd/pomerium/main_test.go index 2ec946c85..68f2a653b 100644 --- a/cmd/pomerium/main_test.go +++ b/cmd/pomerium/main_test.go @@ -7,48 +7,19 @@ import ( "net/http/httptest" "net/url" "os" + "os/signal" "reflect" - "strings" + "syscall" "testing" + "time" + "github.com/google/go-cmp/cmp" "github.com/pomerium/pomerium/internal/config" + "github.com/pomerium/pomerium/internal/httputil" "github.com/pomerium/pomerium/internal/middleware" "google.golang.org/grpc" ) -func Test_startRedirectServer(t *testing.T) { - - tests := []struct { - name string - addr string - want string - wantErr bool - }{ - {"empty", "", "", true}, - {":http", ":http", ":http", false}, - {"localhost:80", "localhost:80", "localhost:80", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := startRedirectServer(tt.addr) - if (err != nil) != tt.wantErr { - t.Errorf("startRedirectServer() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != nil { - defer got.Close() - ts := httptest.NewServer(got.Handler) - defer ts.Close() - _, err := http.Get(ts.URL) - if !strings.Contains(err.Error(), "https") { - t.Errorf("startRedirectServer() = %v, want %v", err, tt.want) - return - } - } - }) - } -} - func Test_newAuthenticateService(t *testing.T) { grpcAuth := middleware.NewSharedSecretCred("test") grpcOpts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcAuth.ValidateRequest)} @@ -193,7 +164,7 @@ func Test_newProxyeService(t *testing.T) { } } -func Test_wrapMiddleware(t *testing.T) { +func Test_mainHandler(t *testing.T) { o := config.Options{ Services: "all", Headers: map[string]string{ @@ -214,7 +185,7 @@ func Test_wrapMiddleware(t *testing.T) { }) mux.Handle("/404", h) - out := wrapMiddleware(&o, mux) + out := mainHandler(&o, mux) out.ServeHTTP(rr, req) expected := fmt.Sprintf("OK") body := rr.Body.String() @@ -223,87 +194,106 @@ func Test_wrapMiddleware(t *testing.T) { t.Errorf("handler returned unexpected body: got %v want %v", body, expected) } } -func Test_parseOptions(t *testing.T) { + +func Test_configToServerOptions(t *testing.T) { tests := []struct { - name string - envKey string - envValue string - servicesEnvKey string - servicesEnvValue string - wantSharedKey string - wantErr bool + name string + opt *config.Options + want *httputil.ServerOptions }{ - {"no shared secret", "", "", "SERVICES", "authenticate", "skip", true}, - {"no shared secret in all mode", "", "", "", "", "", false}, - {"good", "SHARED_SECRET", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", "", "", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", false}, + {"simple convert", &config.Options{Addr: ":http"}, &httputil.ServerOptions{Addr: ":http"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv(tt.servicesEnvKey, tt.servicesEnvValue) - os.Setenv(tt.envKey, tt.envValue) - defer os.Unsetenv(tt.envKey) - defer os.Unsetenv(tt.servicesEnvKey) - - got, err := parseOptions("") - if (err != nil) != tt.wantErr { - t.Errorf("parseOptions() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != nil && got.Services != "all" && got.SharedKey != tt.wantSharedKey { - t.Errorf("parseOptions()\n") - t.Errorf("got: %+v\n", got.SharedKey) - t.Errorf("want: %+v\n", tt.wantSharedKey) - + if diff := cmp.Diff(configToServerOptions(tt.opt), tt.want); diff != "" { + t.Errorf("configToServerOptions() = \n %s", diff) } }) } } -type mockService struct { - fail bool - Updated bool -} - -func (m *mockService) UpdateOptions(o config.Options) error { - - m.Updated = true - if m.fail { - return fmt.Errorf("failed") - } - return nil -} - -func Test_handleConfigUpdate(t *testing.T) { - os.Clearenv() - os.Setenv("SHARED_SECRET", "foo") - defer os.Unsetenv("SHARED_SECRET") - - blankOpts, err := config.NewOptions("https://authenticate.example", "https://authorize.example") - if err != nil { - t.Fatal(err) - } - - goodOpts, err := config.OptionsFromViper("") - if err != nil { - t.Fatal(err) - } +func Test_setupGRPCServer(t *testing.T) { tests := []struct { - name string - service *mockService - oldOpts config.Options - wantUpdate bool + name string + opt *config.Options + dontWant *grpc.Server }{ - {"good", &mockService{fail: false}, *blankOpts, true}, - {"bad", &mockService{fail: true}, *blankOpts, true}, - {"no change", &mockService{fail: false}, *goodOpts, false}, + {"good", &config.Options{SharedKey: "test"}, nil}, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - handleConfigUpdate(&tt.oldOpts, []config.OptionsUpdater{tt.service}) - if tt.service.Updated != tt.wantUpdate { - t.Errorf("Failed to update config on service") + if diff := cmp.Diff(setupGRPCServer(tt.opt), tt.dontWant); diff == "" { + t.Errorf("setupGRPCServer() = \n %s", diff) } }) } } + +func Test_setupTracing(t *testing.T) { + tests := []struct { + name string + opt *config.Options + }{ + {"good jaeger", &config.Options{TracingProvider: "jaeger", TracingJaegerAgentEndpoint: "localhost:0", TracingJaegerCollectorEndpoint: "localhost:0"}}, + {"dont register aything", &config.Options{}}, + {"bad provider", &config.Options{TracingProvider: "bad provider"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setupTracing(tt.opt) + }) + } +} + +func Test_setupMetrics(t *testing.T) { + tests := []struct { + name string + opt *config.Options + }{ + {"dont register aything", &config.Options{}}, + {"good metrics server", &config.Options{MetricsAddr: "localhost:0"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + defer signal.Stop(c) + setupMetrics(tt.opt) + syscall.Kill(syscall.Getpid(), syscall.SIGINT) + waitSig(t, c, syscall.SIGINT) + + }) + } +} + +func Test_setupHTTPRedirectServer(t *testing.T) { + tests := []struct { + name string + opt *config.Options + }{ + {"dont register aything", &config.Options{}}, + {"good redirect server", &config.Options{HTTPRedirectAddr: "localhost:0"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + defer signal.Stop(c) + setupHTTPRedirectServer(tt.opt) + syscall.Kill(syscall.Getpid(), syscall.SIGINT) + waitSig(t, c, syscall.SIGINT) + + }) + } +} + +func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) { + select { + case s := <-c: + if s != sig { + t.Fatalf("signal was %v, want %v", s, sig) + } + case <-time.After(1 * time.Second): + t.Fatalf("timeout waiting for %v", sig) + } +} diff --git a/docs/reference/readme.md b/docs/reference/readme.md index e7a04b049..f9fa8b75b 100644 --- a/docs/reference/readme.md +++ b/docs/reference/readme.md @@ -154,13 +154,17 @@ If set, the HTTP Redirect Address specifies the host and port to redirect http t - Environmental Variable: `METRICS_ADDRESS` - Config File Key: `metrics_address` - Type: `string` -- Example: `:8080`, `127.0.0.1:9090`, `` +- Example: `:9090`, `127.0.0.1:9090` - Default: `disabled` - Optional Expose a prometheus format HTTP endpoint on the specified port. Disabled by default. -**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not expose the metrics port if this is sensitive information. +:::warning + +**Use with caution:** the endpoint can expose frontend and backend server names or addresses. Do not externally expose the metrics if this is sensitive information. + +::: #### Metrics tracked @@ -187,6 +191,38 @@ pomerium_config_last_reload_success | Gauge | Whether the last con pomerium_config_last_reload_success_timestamp | Gauge | The timestamp of the last successful configuration reload by service pomerium_build_info | Gauge | Pomerium build metadata by git revision, service, version and goversion +### Tracing + +Tracing tracks the progression of a single user request as it is handled by Pomerium. + +Each unit work is called a Span in a trace. Spans include metadata about the work, including the time spent in the step (latency), status, time events, attributes, links. You can use tracing to debug errors and latency issues in your applications, including in downstream connections. + +#### Shared Tracing Settings + +Config Key | Description | Required +:--------------- | :---------------------------------------------------------------- | -------- +tracing_provider | The name of the tracing provider. (e.g. jaeger) | ✅ +tracing_debug | Will disable [sampling](https://opencensus.io/tracing/sampling/). | ❌ + +#### Jaeger + +[Jaeger](https://www.jaegertracing.io/) is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and troubleshooting microservices-based distributed systems, including: + +- Distributed context propagation +- Distributed transaction monitoring +- Root cause analysis +- Service dependency analysis +- Performance / latency optimization + +Config Key | Description | Required +:-------------------------------- | :------------------------------------------ | -------- +tracing_jaeger_collector_endpoint | Url to the Jaeger HTTP Thrift collector. | ✅ +tracing_jaeger_agent_endpoint | Send spans to jaeger-agent at this address. | ✅ + +##### Example + +![jaeger example trace](./tracing/jaeger.png) pomerium_config_last_reload_success_timestamp | Gauge | The timestamp of the last successful configuration reload by service pomerium_build_info | Gauge | Pomerium build metadata by git revision, service, version and goversion + ### Policy - Environmental Variable: `POLICY` diff --git a/docs/reference/tracing/jaeger.png b/docs/reference/tracing/jaeger.png new file mode 100644 index 000000000..a60289c33 Binary files /dev/null and b/docs/reference/tracing/jaeger.png differ diff --git a/go.mod b/go.mod index efc2de1fc..0dcdc8412 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( cloud.google.com/go v0.40.0 // indirect + contrib.go.opencensus.io/exporter/jaeger v0.1.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/fsnotify/fsnotify v1.4.7 github.com/golang/mock v1.3.1 diff --git a/go.sum b/go.sum index 0c0d3c340..e06dfd01c 100644 --- a/go.sum +++ b/go.sum @@ -4,13 +4,19 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk= cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= +contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= +contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -28,6 +34,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -36,6 +45,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -49,6 +59,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -57,6 +68,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -67,6 +80,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -91,10 +105,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -105,20 +124,25 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAm github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0= @@ -151,6 +175,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -171,10 +196,12 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -195,11 +222,14 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -216,6 +246,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -226,6 +257,8 @@ golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g= google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= @@ -243,6 +276,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190611190212-a7e196e89fd3 h1:0LGHEA/u5XLibPOx6D7D8FBT/ax6wT57vNKY0QckCwo= google.golang.org/genproto v0.0.0-20190611190212-a7e196e89fd3/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -253,13 +287,16 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/config/helpers.go b/internal/config/helpers.go index d916c55ce..2a23ae2f2 100644 --- a/internal/config/helpers.go +++ b/internal/config/helpers.go @@ -1,16 +1,5 @@ package config // import "github.com/pomerium/pomerium/internal/config" -import "os" - -// findPwd returns best guess at current working directory -func findPwd() string { - p, err := os.Getwd() - if err != nil { - return "." - } - return p -} - // IsValidService checks to see if a service is a valid service mode func IsValidService(s string) bool { switch s { diff --git a/internal/config/options.go b/internal/config/options.go index 2ac8239ce..4162ae154 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -7,11 +7,14 @@ import ( "net/url" "path/filepath" "reflect" + "strconv" "strings" "time" "github.com/pomerium/pomerium/internal/cryptutil" + "github.com/pomerium/pomerium/internal/fileutil" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/metrics" "github.com/pomerium/pomerium/internal/urlutil" "github.com/mitchellh/hashstructure" @@ -129,6 +132,19 @@ type Options struct { // Address/Port to bind to for prometheus metrics MetricsAddr string `mapstructure:"metrics_address"` + + // Tracing shared settings + TracingProvider string `mapstructure:"tracing_provider"` + TracingDebug bool `mapstructure:"tracing_debug"` + + // Jaeger + + // CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector. + // For example, http://localhost:14268/api/traces + TracingJaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint"` + // AgentEndpoint instructs exporter to send spans to jaeger-agent at this address. + // For example, localhost:6831. + TracingJaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint"` } var defaultOptions = Options{ @@ -148,8 +164,8 @@ var defaultOptions = Options{ "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", }, Addr: ":https", - CertFile: filepath.Join(findPwd(), "cert.pem"), - KeyFile: filepath.Join(findPwd(), "privkey.pem"), + CertFile: filepath.Join(fileutil.Getwd(), "cert.pem"), + KeyFile: filepath.Join(fileutil.Getwd(), "privkey.pem"), ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 0, // support streaming by default @@ -339,3 +355,56 @@ func (o *Options) Checksum() string { } return fmt.Sprintf("%x", hash) } + +func ParseOptions(configFile string) (*Options, error) { + o, err := OptionsFromViper(configFile) + if err != nil { + return nil, err + } + if o.Debug { + log.SetDebugMode() + } + if o.LogLevel != "" { + log.SetLevel(o.LogLevel) + } + metrics.AddPolicyCountCallback(o.Services, func() int64 { + return int64(len(o.Policies)) + }) + + checksumDec, err := strconv.ParseUint(o.Checksum(), 16, 64) + if err != nil { + log.Warn().Err(err).Msg("Could not parse config checksum into decimal") + } + metrics.SetConfigChecksum(o.Services, checksumDec) + + return o, nil +} + +func HandleConfigUpdate(configFile string, opt *Options, services []OptionsUpdater) *Options { + newOpt, err := ParseOptions(configFile) + if err != nil { + log.Error().Err(err).Msg("cmd/pomerium: could not reload configuration") + return opt + } + optChecksum := opt.Checksum() + newOptChecksum := newOpt.Checksum() + + log.Debug(). + Str("old-checksum", optChecksum). + Str("new-checksum", newOptChecksum). + Msg("cmd/pomerium: configuration file changed") + + if newOptChecksum == optChecksum { + log.Debug().Msg("cmd/pomerium: loaded configuration has not changed") + return opt + } + + log.Info().Str("checksum", newOptChecksum).Msg("cmd/pomerium: checksum changed") + for _, service := range services { + if err := service.UpdateOptions(*newOpt); err != nil { + log.Error().Err(err).Msg("cmd/pomerium: could not update options") + } + } + + return newOpt +} diff --git a/internal/config/options_test.go b/internal/config/options_test.go index fa982daed..69f3e132e 100644 --- a/internal/config/options_test.go +++ b/internal/config/options_test.go @@ -408,3 +408,99 @@ func TestOptionsFromViper(t *testing.T) { }) } } + +func Test_parseOptions(t *testing.T) { + viper.Reset() + + tests := []struct { + name string + envKey string + envValue string + servicesEnvKey string + servicesEnvValue string + wantSharedKey string + wantErr bool + }{ + {"no shared secret", "", "", "SERVICES", "authenticate", "skip", true}, + {"no shared secret in all mode", "", "", "", "", "", false}, + {"good", "SHARED_SECRET", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", "", "", "YixWi1MYh77NMECGGIJQevoonYtVF+ZPRkQZrrmeRqM=", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv(tt.servicesEnvKey, tt.servicesEnvValue) + os.Setenv(tt.envKey, tt.envValue) + defer os.Unsetenv(tt.envKey) + defer os.Unsetenv(tt.servicesEnvKey) + + got, err := ParseOptions("") + if (err != nil) != tt.wantErr { + t.Errorf("ParseOptions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != nil && got.Services != "all" && got.SharedKey != tt.wantSharedKey { + t.Errorf("ParseOptions()\n") + t.Errorf("got: %+v\n", got.SharedKey) + t.Errorf("want: %+v\n", tt.wantSharedKey) + + } + }) + } +} + +type mockService struct { + fail bool + Updated bool +} + +func (m *mockService) UpdateOptions(o Options) error { + + m.Updated = true + if m.fail { + return fmt.Errorf("failed") + } + return nil +} + +func Test_HandleConfigUpdate(t *testing.T) { + os.Clearenv() + os.Setenv("SHARED_SECRET", "foo") + defer os.Unsetenv("SHARED_SECRET") + + blankOpts, err := NewOptions("https://authenticate.example", "https://authorize.example") + if err != nil { + t.Fatal(err) + } + + goodOpts, err := OptionsFromViper("") + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + envarKey string + envarValue string + service *mockService + oldOpts Options + wantUpdate bool + }{ + {"good", "", "", &mockService{fail: false}, *blankOpts, true}, + {"good set debug", "POMERIUM_DEBUG", "true", &mockService{fail: false}, *blankOpts, true}, + {"bad", "", "", &mockService{fail: true}, *blankOpts, true}, + {"no change", "", "", &mockService{fail: false}, *goodOpts, false}, + {"bad policy file unmarshal error", "POLICY", base64.StdEncoding.EncodeToString([]byte("{json:}")), &mockService{fail: false}, *blankOpts, false}, + {"bad header key", "SERVICES", "error", &mockService{fail: false}, *blankOpts, false}, + {"bad header header value", "HEADERS", "x;y;z", &mockService{fail: false}, *blankOpts, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv(tt.envarKey, tt.envarValue) + defer os.Unsetenv(tt.envarKey) + + HandleConfigUpdate("", &tt.oldOpts, []OptionsUpdater{tt.service}) + if tt.service.Updated != tt.wantUpdate { + t.Errorf("Failed to update config on service") + } + }) + } +} diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go index 904dd38e0..a7b0bba7e 100644 --- a/internal/fileutil/fileutil.go +++ b/internal/fileutil/fileutil.go @@ -30,3 +30,17 @@ func IsReadableFile(path string) (bool, error) { fd.Close() return true, nil // Item exists and is readable. } + +// Getwd returns a rooted path name corresponding to the +// current directory. If the current directory can be +// reached via multiple paths (due to symbolic links), +// Getwd may return any one of them. +// +// On failure, will return "." +func Getwd() string { + p, err := os.Getwd() + if err != nil { + return "." + } + return p +} diff --git a/internal/fileutil/fileutil_test.go b/internal/fileutil/fileutil_test.go index 9a8e10380..4ec0f5c9a 100644 --- a/internal/fileutil/fileutil_test.go +++ b/internal/fileutil/fileutil_test.go @@ -1,6 +1,9 @@ -package fileutil // import "github.com/pomerium/pomerium/internal/fileutil" +package fileutil -import "testing" +import ( + "strings" + "testing" +) func TestIsReadableFile(t *testing.T) { @@ -27,3 +30,19 @@ func TestIsReadableFile(t *testing.T) { }) } } + +func TestGetwd(t *testing.T) { + tests := []struct { + name string + want string + }{ + {"most basic example", "internal/fileutil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Getwd(); strings.Contains(tt.want, got) { + t.Errorf("Getwd() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/httputil/errors.go b/internal/httputil/errors.go index 607ad7e21..527d6d9fe 100644 --- a/internal/httputil/errors.go +++ b/internal/httputil/errors.go @@ -23,21 +23,11 @@ func (h Error) Error() string { return fmt.Sprintf("%d %s: %s", h.Code, http.StatusText(h.Code), h.Message) } -// CodeForError maps an error type and returns a corresponding http.Status -func CodeForError(err error) int { - switch err { - case ErrTokenRevoked: - return http.StatusUnauthorized - } - return http.StatusInternalServerError -} - // ErrorResponse renders an error page for errors given a message and a status code. // If no message is passed, defaults to the text of the status code. func ErrorResponse(rw http.ResponseWriter, r *http.Request, e *Error) { - requestID := "" - id, ok := log.IDFromRequest(r) - if ok { + var requestID string + if id, ok := log.IDFromRequest(r); ok { requestID = id } if r.Header.Get("Accept") == "application/json" { diff --git a/internal/httputil/errors_test.go b/internal/httputil/errors_test.go new file mode 100644 index 000000000..7134109f9 --- /dev/null +++ b/internal/httputil/errors_test.go @@ -0,0 +1,49 @@ +package httputil + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestErrorResponse(t *testing.T) { + tests := []struct { + name string + rw http.ResponseWriter + r *http.Request + e *Error + }{ + {"good", httptest.NewRecorder(), &http.Request{Method: http.MethodGet}, &Error{Code: http.StatusBadRequest, Message: "missing id token"}}, + {"good json", httptest.NewRecorder(), &http.Request{Method: http.MethodGet, Header: http.Header{"Accept": []string{"application/json"}}}, &Error{Code: http.StatusBadRequest, Message: "missing id token"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ErrorResponse(tt.rw, tt.r, tt.e) + }) + } +} + +func TestError_Error(t *testing.T) { + + tests := []struct { + name string + Message string + Code int + CanDebug bool + want string + }{ + {"good", "short and stout", http.StatusTeapot, false, "418 I'm a teapot: short and stout"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := Error{ + Message: tt.Message, + Code: tt.Code, + CanDebug: tt.CanDebug, + } + if got := h.Error(); got != tt.want { + t.Errorf("Error.Error() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/httputil/http.go b/internal/httputil/http.go new file mode 100644 index 000000000..289cb792f --- /dev/null +++ b/internal/httputil/http.go @@ -0,0 +1,76 @@ +package httputil // import "github.com/pomerium/pomerium/internal/httputil" + +import ( + "context" + "fmt" + stdlog "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/urlutil" +) + +// NewHTTPServer starts a http server given a set of options and a handler. +// +// It is the caller's responsibility to Close() or Shutdown() the returned +// server. +func NewHTTPServer(opt *ServerOptions, h http.Handler) *http.Server { + if opt == nil { + opt = defaultHTTPServerOptions + } else { + opt.applyHTTPDefaults() + } + sublogger := log.With().Str("addr", opt.Addr).Logger() + srv := http.Server{ + Addr: opt.Addr, + ReadHeaderTimeout: opt.ReadHeaderTimeout, + ReadTimeout: opt.ReadTimeout, + WriteTimeout: opt.WriteTimeout, + IdleTimeout: opt.IdleTimeout, + Handler: h, + ErrorLog: stdlog.New(&log.StdLogWrapper{Logger: &sublogger}, "", 0), + } + + go func() { + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + log.Error().Str("addr", opt.Addr).Err(err).Msg("internal/httputil: unexpected shutdown") + } + }() + return &srv +} + +func RedirectHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Connection", "close") + url := fmt.Sprintf("https://%s%s", urlutil.StripPort(r.Host), r.URL.String()) + http.Redirect(w, r, url, http.StatusMovedPermanently) + }) +} + +// Shutdown attempts to shut down the server when a os interrupt or sigterm +// signal are received without interrupting any +// active connections. Shutdown works by first closing all open +// listeners, then closing all idle connections, and then waiting +// indefinitely for connections to return to idle and then shut down. +// If the provided context expires before the shutdown is complete, +// Shutdown returns the context's error, otherwise it returns any +// error returned from closing the Server's underlying Listener(s). +// +// When Shutdown is called, Serve, ListenAndServe, and +// ListenAndServeTLS immediately return ErrServerClosed. +func Shutdown(srv *http.Server) { + sigint := make(chan os.Signal, 1) + signal.Notify(sigint, os.Interrupt) + signal.Notify(sigint, syscall.SIGTERM) + rec := <-sigint + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + log.Info().Str("signal", rec.String()).Msg("internal/httputil: shutting down servers") + if err := srv.Shutdown(ctx); err != nil { + log.Error().Err(err).Msg("internal/httputil: shutdown failed") + } +} diff --git a/internal/httputil/http_test.go b/internal/httputil/http_test.go new file mode 100644 index 000000000..ec67af50b --- /dev/null +++ b/internal/httputil/http_test.go @@ -0,0 +1,49 @@ +package httputil + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNewHTTPServer(t *testing.T) { + tests := []struct { + name string + opts *ServerOptions + // wantErr bool + }{ + {"localhost:9232", &ServerOptions{Addr: "localhost:9232"}}, + {"localhost:65536", &ServerOptions{Addr: "localhost:-1"}}, // will fail, but won't err + {"empty", &ServerOptions{}}, + {"empty", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := NewHTTPServer(tt.opts, RedirectHandler()) + + defer srv.Close() + + // we cheat a little bit here and use the httptest server to test the client + ts := httptest.NewServer(srv.Handler) + defer ts.Close() + client := ts.Client() + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + res, err := client.Get(ts.URL) + if err != nil { + log.Fatal(err) + } + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s", greeting) + + }) + } +} diff --git a/internal/httputil/options.go b/internal/httputil/options.go new file mode 100644 index 000000000..4af4df913 --- /dev/null +++ b/internal/httputil/options.go @@ -0,0 +1,87 @@ +package httputil // import "github.com/pomerium/pomerium/internal/httputil" + +import ( + "path/filepath" + "time" + + "github.com/pomerium/pomerium/internal/fileutil" +) + +// ServerOptions contains the configurations settings for a http server. +type ServerOptions struct { + // Addr specifies the host and port on which the server should serve + // HTTPS requests. If empty, ":https" is used. + Addr string + + // TLS certificates to use. + Cert string + Key string + CertFile string + KeyFile string + + // Timeouts + ReadHeaderTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration +} + +var defaultTLSServerOptions = &ServerOptions{ + Addr: ":https", + CertFile: filepath.Join(fileutil.Getwd(), "cert.pem"), + KeyFile: filepath.Join(fileutil.Getwd(), "privkey.pem"), + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 0, // support streaming by default + IdleTimeout: 5 * time.Minute, +} + +func (o *ServerOptions) applyTLSDefaults() { + if o.Addr == "" { + o.Addr = defaultTLSServerOptions.Addr + } + if o.Cert == "" && o.CertFile == "" { + o.CertFile = defaultTLSServerOptions.CertFile + } + if o.Key == "" && o.KeyFile == "" { + o.KeyFile = defaultTLSServerOptions.KeyFile + } + if o.ReadHeaderTimeout == 0 { + o.ReadHeaderTimeout = defaultTLSServerOptions.ReadHeaderTimeout + } + if o.ReadTimeout == 0 { + o.ReadTimeout = defaultTLSServerOptions.ReadTimeout + } + if o.WriteTimeout == 0 { + o.WriteTimeout = defaultTLSServerOptions.WriteTimeout + } + if o.IdleTimeout == 0 { + o.IdleTimeout = defaultTLSServerOptions.IdleTimeout + } +} + +var defaultHTTPServerOptions = &ServerOptions{ + Addr: ":http", + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + IdleTimeout: 5 * time.Minute, +} + +func (o *ServerOptions) applyHTTPDefaults() { + if o.Addr == "" { + o.Addr = defaultHTTPServerOptions.Addr + } + if o.ReadHeaderTimeout == 0 { + o.ReadHeaderTimeout = defaultHTTPServerOptions.ReadHeaderTimeout + } + if o.ReadTimeout == 0 { + o.ReadTimeout = defaultHTTPServerOptions.ReadTimeout + } + if o.WriteTimeout == 0 { + o.WriteTimeout = defaultHTTPServerOptions.WriteTimeout + } + if o.IdleTimeout == 0 { + o.IdleTimeout = defaultHTTPServerOptions.IdleTimeout + } +} diff --git a/internal/httputil/test_data/cert.pem b/internal/httputil/test_data/cert.pem new file mode 100644 index 000000000..87a6bf824 --- /dev/null +++ b/internal/httputil/test_data/cert.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBeDCCAR+gAwIBAgIUUGE8w2S7XzpkVLbNq5QUxyVOwqEwCgYIKoZIzj0EAwIw +ETEPMA0GA1UEAwwGdW51c2VkMCAXDTE5MDcxNTIzNDQyOVoYDzQ3NTcwNjExMjM0 +NDI5WjARMQ8wDQYDVQQDDAZ1bnVzZWQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC +AAQW6Z1KsR712c8RRTcu7ILyXowzo9582ClKxEvgasPbZchMyOoMoWuOolN/QWjV +labi/4R2zqzzyuwvMQL5wotFo1MwUTAdBgNVHQ4EFgQURYdcaniRqBHXeaM79LtV +pyJ4EwAwHwYDVR0jBBgwFoAURYdcaniRqBHXeaM79LtVpyJ4EwAwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBHbhVnGbwXqaMZ1dB8eBAK56jyeWDZ +2PWXmFMTu7+RywIgaZ7UwVNB2k7KjEEBiLm0PIRcpJmczI2cP9+ZMIkPHHw= +-----END CERTIFICATE----- diff --git a/internal/httputil/test_data/privkey.pem b/internal/httputil/test_data/privkey.pem new file mode 100644 index 000000000..287f2b874 --- /dev/null +++ b/internal/httputil/test_data/privkey.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMQiDy26/R4ca/OdnjIf8OEDeHcw8yB5SDV9FD500CW5oAoGCCqGSM49 +AwEHoUQDQgAEFumdSrEe9dnPEUU3LuyC8l6MM6PefNgpSsRL4GrD22XITMjqDKFr +jqJTf0Fo1ZWm4v+Eds6s88rsLzEC+cKLRQ== +-----END EC PRIVATE KEY----- diff --git a/internal/httputil/https.go b/internal/httputil/tls.go similarity index 67% rename from internal/httputil/https.go rename to internal/httputil/tls.go index 215c94ba9..600ebf572 100644 --- a/internal/httputil/https.go +++ b/internal/httputil/tls.go @@ -7,83 +7,20 @@ import ( stdlog "log" "net" "net/http" - "os" - "path/filepath" "strings" - "time" "github.com/pomerium/pomerium/internal/fileutil" "github.com/pomerium/pomerium/internal/log" ) -// Options contains the configurations settings for a TLS http server. -type Options struct { - // Addr specifies the host and port on which the server should serve - // HTTPS requests. If empty, ":https" is used. - Addr string - - // TLS certificates to use. - Cert string - Key string - CertFile string - KeyFile string - - // Timeouts - ReadHeaderTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - IdleTimeout time.Duration -} - -var defaultOptions = &Options{ - Addr: ":https", - CertFile: filepath.Join(findKeyDir(), "cert.pem"), - KeyFile: filepath.Join(findKeyDir(), "privkey.pem"), - ReadHeaderTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 0, // support streaming by default - IdleTimeout: 5 * time.Minute, -} - -func findKeyDir() string { - p, err := os.Getwd() - if err != nil { - return "." - } - return p -} - -func (o *Options) applyDefaults() { - if o.Addr == "" { - o.Addr = defaultOptions.Addr - } - if o.Cert == "" && o.CertFile == "" { - o.CertFile = defaultOptions.CertFile - } - if o.Key == "" && o.KeyFile == "" { - o.KeyFile = defaultOptions.KeyFile - } - if o.ReadHeaderTimeout == 0 { - o.ReadHeaderTimeout = defaultOptions.ReadHeaderTimeout - } - if o.ReadTimeout == 0 { - o.ReadTimeout = defaultOptions.ReadTimeout - } - if o.WriteTimeout == 0 { - o.WriteTimeout = defaultOptions.WriteTimeout - } - if o.IdleTimeout == 0 { - o.IdleTimeout = defaultOptions.IdleTimeout - } -} - -// ListenAndServeTLS serves the provided handlers by HTTPS -// using the provided options. -func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http.Handler) error { +// NewTLSServer creates a new TLS server given a set of options, handlers, and +// optionally a set of gRPC endpoints as well. +// It is the callers responsibility to close the resturned server. +func NewTLSServer(opt *ServerOptions, httpHandler http.Handler, grpcHandler http.Handler) (*http.Server, error) { if opt == nil { - opt = defaultOptions + opt = defaultTLSServerOptions } else { - opt.applyDefaults() + opt.applyTLSDefaults() } var cert *tls.Certificate var err error @@ -93,12 +30,12 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http. cert, err = readCertificateFile(opt.CertFile, opt.KeyFile) } if err != nil { - return fmt.Errorf("https: failed loading x509 certificate: %v", err) + return nil, fmt.Errorf("internal/httputil: failed loading x509 certificate: %v", err) } config := newDefaultTLSConfig(cert) ln, err := net.Listen("tcp", opt.Addr) if err != nil { - return err + return nil, err } ln = tls.NewListener(ln, config) @@ -112,7 +49,7 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http. sublogger := log.With().Str("addr", opt.Addr).Logger() // Set up the main server. - server := &http.Server{ + srv := &http.Server{ ReadHeaderTimeout: opt.ReadHeaderTimeout, ReadTimeout: opt.ReadTimeout, WriteTimeout: opt.WriteTimeout, @@ -121,8 +58,13 @@ func ListenAndServeTLS(opt *Options, httpHandler http.Handler, grpcHandler http. Handler: h, ErrorLog: stdlog.New(&log.StdLogWrapper{Logger: &sublogger}, "", 0), } + go func() { + if err := srv.Serve(ln); err != http.ErrServerClosed { + log.Error().Err(err).Msg("internal/httputil: tls server crashed") + } + }() - return server.Serve(ln) + return srv, nil } func decodeCertificate(cert, key string) (*tls.Certificate, error) { @@ -189,8 +131,8 @@ func newDefaultTLSConfig(cert *tls.Certificate) *tls.Config { return tlsConfig } -// grpcHandlerFunc splits request serving between gRPC and HTTPS depending on the request type. -// Requires HTTP/2. +// grpcHandlerFunc splits request serving between gRPC and HTTPS depending on +// the request type. Requires HTTP/2 to be enabled. func grpcHandlerFunc(rpcServer http.Handler, other http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") diff --git a/internal/httputil/tls_test.go b/internal/httputil/tls_test.go new file mode 100644 index 000000000..d9a2c22e2 --- /dev/null +++ b/internal/httputil/tls_test.go @@ -0,0 +1,210 @@ +package httputil + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "os" + "os/signal" + "syscall" + "testing" + "time" +) + +const privKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMQiDy26/R4ca/OdnjIf8OEDeHcw8yB5SDV9FD500CW5oAoGCCqGSM49 +AwEHoUQDQgAEFumdSrEe9dnPEUU3LuyC8l6MM6PefNgpSsRL4GrD22XITMjqDKFr +jqJTf0Fo1ZWm4v+Eds6s88rsLzEC+cKLRQ== +-----END EC PRIVATE KEY-----` +const pubKey = `-----BEGIN CERTIFICATE----- +MIIBeDCCAR+gAwIBAgIUUGE8w2S7XzpkVLbNq5QUxyVOwqEwCgYIKoZIzj0EAwIw +ETEPMA0GA1UEAwwGdW51c2VkMCAXDTE5MDcxNTIzNDQyOVoYDzQ3NTcwNjExMjM0 +NDI5WjARMQ8wDQYDVQQDDAZ1bnVzZWQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC +AAQW6Z1KsR712c8RRTcu7ILyXowzo9582ClKxEvgasPbZchMyOoMoWuOolN/QWjV +labi/4R2zqzzyuwvMQL5wotFo1MwUTAdBgNVHQ4EFgQURYdcaniRqBHXeaM79LtV +pyJ4EwAwHwYDVR0jBBgwFoAURYdcaniRqBHXeaM79LtVpyJ4EwAwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBHbhVnGbwXqaMZ1dB8eBAK56jyeWDZ +2PWXmFMTu7+RywIgaZ7UwVNB2k7KjEEBiLm0PIRcpJmczI2cP9+ZMIkPHHw= +-----END CERTIFICATE-----` + +func TestNewTLSServer(t *testing.T) { + t.Parallel() + tests := []struct { + name string + opt *ServerOptions + httpHandler http.Handler + grpcHandler http.Handler + // want *http.Server + wantErr bool + }{ + {"good basic http handler", + &ServerOptions{ + Addr: "127.0.0.1:0", + Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)), + Key: base64.StdEncoding.EncodeToString([]byte(privKey)), + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + false}, + {"good basic http and grpc handler", + &ServerOptions{ + Addr: "127.0.0.1:0", + Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)), + Key: base64.StdEncoding.EncodeToString([]byte(privKey)), + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, grpc") + }), + false}, + {"good with cert files", + &ServerOptions{ + Addr: "127.0.0.1:0", + CertFile: "test_data/cert.pem", + KeyFile: "test_data/privkey.pem", + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, grpc") + }), + false}, + {"unreadable cert file", + &ServerOptions{ + Addr: "127.0.0.1:0", + CertFile: "test_data", + KeyFile: "test_data/privkey.pem", + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, grpc") + }), + true}, + {"unreadable key file", + &ServerOptions{ + Addr: "127.0.0.1:0", + CertFile: "./test_data/cert.pem", + KeyFile: "./test_data", + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, grpc") + }), + true}, + {"unreadable key file", + &ServerOptions{ + Addr: "127.0.0.1:0", + CertFile: "./test_data/cert.pem", + KeyFile: "./test_data/file-does-not-exist", + }, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, grpc") + }), + true}, + {"bad private key base64", + &ServerOptions{ + Addr: "127.0.0.1:0", + Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)), + Key: "bad guy", + }, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + true}, + {"bad public key base64", + &ServerOptions{ + Addr: "127.0.0.1:9999", + Key: base64.StdEncoding.EncodeToString([]byte(pubKey)), + Cert: "bad guy", + }, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + true}, + {"bad port - invalid port range ", + &ServerOptions{ + Addr: "127.0.0.1:65536", + Cert: base64.StdEncoding.EncodeToString([]byte(pubKey)), + Key: base64.StdEncoding.EncodeToString([]byte(privKey)), + }, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + true}, + {"nil apply default but will fail", + nil, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + true}, + {"empty, apply defaults to missing", + &ServerOptions{}, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, http") + }), + nil, + true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv, err := NewTLSServer(tt.opt, tt.httpHandler, tt.grpcHandler) + if (err != nil) != tt.wantErr { + t.Errorf("NewTLSServer() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil { + // we cheat a little bit here and use the httptest server to test the client + ts := httptest.NewTLSServer(srv.Handler) + defer ts.Close() + client := ts.Client() + res, err := client.Get(ts.URL) + if err != nil { + log.Fatal(err) + } + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s", greeting) + } + if srv != nil { + // simulate a sigterm and cleanup the server + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGINT) + defer signal.Stop(c) + go Shutdown(srv) + syscall.Kill(syscall.Getpid(), syscall.SIGINT) + waitSig(t, c, syscall.SIGINT) + } + + }) + } +} +func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) { + select { + case s := <-c: + if s != sig { + t.Fatalf("signal was %v, want %v", s, sig) + } + case <-time.After(1 * time.Second): + t.Fatalf("timeout waiting for %v", sig) + } +} diff --git a/internal/identity/providers.go b/internal/identity/providers.go index f2d21a00f..394e69566 100644 --- a/internal/identity/providers.go +++ b/internal/identity/providers.go @@ -9,11 +9,12 @@ import ( "net/url" "time" - oidc "github.com/pomerium/go-oidc" - "golang.org/x/oauth2" - "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" + + oidc "github.com/pomerium/go-oidc" + "golang.org/x/oauth2" ) const ( @@ -117,6 +118,8 @@ func (p *Provider) GetSignInURL(state string) string { // Validate does NOT check if revoked. // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation func (p *Provider) Validate(ctx context.Context, idToken string) (bool, error) { + ctx, span := trace.StartSpan(ctx, "identity.provider.Validate") + defer span.End() _, err := p.verifier.Verify(ctx, idToken) if err != nil { log.Error().Err(err).Msg("identity: failed to verify session state") diff --git a/internal/metrics/exporter.go b/internal/metrics/exporter.go deleted file mode 100644 index 832cd8363..000000000 --- a/internal/metrics/exporter.go +++ /dev/null @@ -1,30 +0,0 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" - -import ( - "net/http" - - ocProm "contrib.go.opencensus.io/exporter/prometheus" - prom "github.com/prometheus/client_golang/prometheus" - "go.opencensus.io/stats/view" -) - -//NewPromHTTPListener creates a prometheus exporter on ListenAddr -func NewPromHTTPListener(addr string) error { - return http.ListenAndServe(addr, newPromHTTPHandler()) -} - -// newPromHTTPHandler creates a new prometheus exporter handler for /metrics -func newPromHTTPHandler() http.Handler { - // TODO this is a cheap way to get thorough go process - // stats. It will not work with additional exporters. - // It should turn into an FR to the OC framework - reg := prom.DefaultRegisterer.(*prom.Registry) - pe, _ := ocProm.NewExporter(ocProm.Options{ - Namespace: "pomerium", - Registry: reg, - }) - view.RegisterExporter(pe) - mux := http.NewServeMux() - mux.Handle("/metrics", pe) - return mux -} diff --git a/internal/metrics/middleware.go b/internal/metrics/middleware.go deleted file mode 100644 index 283cd25a6..000000000 --- a/internal/metrics/middleware.go +++ /dev/null @@ -1,151 +0,0 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" - -import ( - "net/http" - - "go.opencensus.io/plugin/ochttp" - - "github.com/pomerium/pomerium/internal/log" - - "github.com/pomerium/pomerium/internal/tripper" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" -) - -var ( - httpSizeDistribution = view.Distribution( - 1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, 524288, - 1048576, 2097152, 4194304, 8388608, - ) - - httpLatencyDistrubtion = view.Distribution( - 1, 2, 5, 7, 10, 25, 500, 750, - 100, 250, 500, 750, - 1000, 2500, 5000, 7500, - 10000, 25000, 50000, 75000, - 100000, - ) - - // httpClientRequestCount = stats.Int64("http_client_requests_total", "Total HTTP Client Requests", "1") - // httpClientResponseSize = stats.Int64("http_client_response_size_bytes", "HTTP Client Response Size in bytes", "bytes") - // httpClientRequestDuration = stats.Int64("http_client_request_duration_ms", "HTTP Client Request duration in ms", "ms") - - // HTTPServerRequestCountView is an OpenCensus View that tracks HTTP server requests by pomerium service, host, method and status - HTTPServerRequestCountView = &view.View{ - Name: "http_server_requests_total", - Measure: ochttp.ServerLatency, - Description: "Total HTTP Requests", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode}, - Aggregation: view.Count(), - } - - // HTTPServerRequestDurationView is an OpenCensus view that tracks HTTP server request duration by pomerium service, host, method and status - HTTPServerRequestDurationView = &view.View{ - Name: "http_server_request_duration_ms", - Measure: ochttp.ServerLatency, - Description: "HTTP Request duration in ms", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode}, - Aggregation: httpLatencyDistrubtion, - } - - // HTTPServerRequestSizeView is an OpenCensus view that tracks HTTP server request size by pomerium service, host and method - HTTPServerRequestSizeView = &view.View{ - Name: "http_server_request_size_bytes", - Measure: ochttp.ServerRequestBytes, - Description: "HTTP Server Request Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod}, - Aggregation: httpSizeDistribution, - } - - // HTTPServerResponseSizeView is an OpenCensus view that tracks HTTP server response size by pomerium service, host, method and status - HTTPServerResponseSizeView = &view.View{ - Name: "http_server_response_size_bytes", - Measure: ochttp.ServerResponseBytes, - Description: "HTTP Server Response Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode}, - Aggregation: httpSizeDistribution, - } - - // HTTPClientRequestCountView is an OpenCensus View that tracks HTTP client requests by pomerium service, destination, host, method and status - HTTPClientRequestCountView = &view.View{ - Name: "http_client_requests_total", - Measure: ochttp.ClientRoundtripLatency, - Description: "Total HTTP Client Requests", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination}, - Aggregation: view.Count(), - } - - // HTTPClientRequestDurationView is an OpenCensus view that tracks HTTP client request duration by pomerium service, destination, host, method and status - HTTPClientRequestDurationView = &view.View{ - Name: "http_client_request_duration_ms", - Measure: ochttp.ClientRoundtripLatency, - Description: "HTTP Client Request duration in ms", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination}, - Aggregation: httpLatencyDistrubtion, - } - - // HTTPClientResponseSizeView is an OpenCensus view that tracks HTTP client response size by pomerium service, destination, host, method and status - HTTPClientResponseSizeView = &view.View{ - Name: "http_client_response_size_bytes", - Measure: ochttp.ClientReceivedBytes, - Description: "HTTP Client Response Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, ochttp.StatusCode, keyDestination}, - Aggregation: httpSizeDistribution, - } - - // HTTPClientRequestSizeView is an OpenCensus view that tracks HTTP client request size by pomerium service, destination, host and method - HTTPClientRequestSizeView = &view.View{ - Name: "http_client_response_size_bytes", - Measure: ochttp.ClientSentBytes, - Description: "HTTP Client Response Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyHTTPMethod, keyDestination}, - Aggregation: httpSizeDistribution, - } -) - -// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests -func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler { - - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx, tagErr := tag.New( - r.Context(), - tag.Insert(keyService, service), - tag.Insert(keyHost, r.Host), - tag.Insert(keyHTTPMethod, r.Method), - ) - if tagErr != nil { - log.Warn().Err(tagErr).Str("context", "HTTPMetricsHandler").Msg("internal/metrics: Failed to create metrics context tag") - next.ServeHTTP(w, r) - return - } - - ocHandler := ochttp.Handler{Handler: next} - ocHandler.ServeHTTP(w, r.WithContext(ctx)) - }) - } -} - -// HTTPMetricsRoundTripper creates a metrics tracking tripper for outbound HTTP Requests -func HTTPMetricsRoundTripper(service string, destination string) func(next http.RoundTripper) http.RoundTripper { - return func(next http.RoundTripper) http.RoundTripper { - return tripper.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { - - ctx, tagErr := tag.New( - r.Context(), - tag.Insert(keyService, service), - tag.Insert(keyHost, r.Host), - tag.Insert(keyHTTPMethod, r.Method), - tag.Insert(keyDestination, destination), - ) - - if tagErr != nil { - log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("internal/metrics: Failed to create context tag") - return next.RoundTrip(r) - } - - ocTransport := ochttp.Transport{Base: next} - return ocTransport.RoundTrip(r.WithContext(ctx)) - }) - } -} diff --git a/internal/metrics/tags.go b/internal/metrics/tags.go deleted file mode 100644 index 4d712cc7e..000000000 --- a/internal/metrics/tags.go +++ /dev/null @@ -1,14 +0,0 @@ -package metrics - -import ( - "go.opencensus.io/tag" -) - -var ( - keyHTTPMethod tag.Key = tag.MustNewKey("http_method") - keyService tag.Key = tag.MustNewKey("service") - keyGRPCService tag.Key = tag.MustNewKey("grpc_service") - keyGRPCMethod tag.Key = tag.MustNewKey("grpc_method") - keyHost tag.Key = tag.MustNewKey("host") - keyDestination tag.Key = tag.MustNewKey("destination") -) diff --git a/internal/metrics/view.go b/internal/metrics/view.go deleted file mode 100644 index 5331d8620..000000000 --- a/internal/metrics/view.go +++ /dev/null @@ -1,32 +0,0 @@ -package metrics - -import ( - "github.com/pomerium/pomerium/internal/log" - "go.opencensus.io/stats/view" -) - -var ( - // HTTPClientViews contains opencensus views for HTTP Client metrics - HTTPClientViews = []*view.View{HTTPClientRequestCountView, HTTPClientRequestDurationView, HTTPClientResponseSizeView} - // HTTPServerViews contains opencensus views for HTTP Server metrics - HTTPServerViews = []*view.View{HTTPServerRequestCountView, HTTPServerRequestDurationView, HTTPServerRequestSizeView, HTTPServerResponseSizeView} - // GRPCClientViews contains opencensus views for GRPC Client metrics - GRPCClientViews = []*view.View{GRPCClientRequestCountView, GRPCClientRequestDurationView, GRPCClientResponseSizeView, GRPCClientRequestSizeView} - // GRPCServerViews contains opencensus views for GRPC Server metrics - GRPCServerViews = []*view.View{GRPCServerRequestCountView, GRPCServerRequestDurationView, GRPCServerResponseSizeView, GRPCServerRequestSizeView} - // InfoViews contains opencensus views for Info metrics - InfoViews = []*view.View{ConfigLastReloadView, ConfigLastReloadSuccessView} -) - -// RegisterView registers one of the defined metrics views. It must be called for metrics to see metrics -// in the configured exporters -func RegisterView(v []*view.View) { - if err := view.Register(v...); err != nil { - log.Warn().Str("context", "RegisterView").Err(err).Msg("internal/metrics: Could not register view") - } -} - -// UnRegisterView unregisters one of the defined metrics views. -func UnRegisterView(v []*view.View) { - view.Unregister(v...) -} diff --git a/internal/metrics/view_test.go b/internal/metrics/view_test.go deleted file mode 100644 index 3d34ea85d..000000000 --- a/internal/metrics/view_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package metrics - -import ( - "testing" - - "go.opencensus.io/stats/view" -) - -func Test_RegisterView(t *testing.T) { - RegisterView(HTTPClientViews) - for _, v := range HTTPClientViews { - if view.Find(v.Name) != v { - t.Errorf("Failed to find registered view %s", v.Name) - } - } -} - -func Test_UnregisterView(t *testing.T) { - UnRegisterView(HTTPClientViews) - for _, v := range HTTPClientViews { - if view.Find(v.Name) == v { - t.Errorf("Found unregistered view %s", v.Name) - } - } -} diff --git a/internal/middleware/grpc.go b/internal/middleware/grpc.go index 60324f252..91adf8c5c 100644 --- a/internal/middleware/grpc.go +++ b/internal/middleware/grpc.go @@ -3,6 +3,8 @@ package middleware // import "github.com/pomerium/pomerium/internal/middleware" import ( "context" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -30,6 +32,9 @@ func (s SharedSecretCred) RequireTransportSecurity() bool { return false } // handler and returns an error. Otherwise, the interceptor invokes the unary // handler. func (s SharedSecretCred) ValidateRequest(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ctx, span := trace.StartSpan(ctx, "middleware.grpc.ValidateRequest") + defer span.End() + md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil, status.Errorf(codes.InvalidArgument, "missing metadata") diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index 18e65b1ab..72e4bf70c 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -12,6 +12,8 @@ import ( "github.com/pomerium/pomerium/internal/cryptutil" "github.com/pomerium/pomerium/internal/httputil" + "github.com/pomerium/pomerium/internal/telemetry/trace" + "golang.org/x/net/publicsuffix" ) @@ -19,10 +21,12 @@ import ( func SetHeaders(securityHeaders map[string]string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.SetHeaders") + defer span.End() for key, val := range securityHeaders { w.Header().Set(key, val) } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -32,6 +36,9 @@ func SetHeaders(securityHeaders map[string]string) func(next http.Handler) http. func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateClientSecret") + defer span.End() + if err := r.ParseForm(); err != nil { httpErr := &httputil.Error{Message: err.Error(), Code: http.StatusBadRequest} httputil.ErrorResponse(w, r, httpErr) @@ -47,7 +54,7 @@ func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Hand httputil.ErrorResponse(w, r, &httputil.Error{Code: http.StatusInternalServerError}) return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -57,6 +64,8 @@ func ValidateClientSecret(sharedSecret string) func(next http.Handler) http.Hand func ValidateRedirectURI(rootDomain *url.URL) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateRedirectURI") + defer span.End() err := r.ParseForm() if err != nil { httpErr := &httputil.Error{ @@ -80,7 +89,7 @@ func ValidateRedirectURI(rootDomain *url.URL) func(next http.Handler) http.Handl httputil.ErrorResponse(w, r, httpErr) return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -103,6 +112,9 @@ func SameDomain(u, j *url.URL) bool { func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateSignature") + defer span.End() + err := r.ParseForm() if err != nil { httpErr := &httputil.Error{Message: err.Error(), Code: http.StatusBadRequest} @@ -120,7 +132,7 @@ func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -129,11 +141,14 @@ func ValidateSignature(sharedSecret string) func(next http.Handler) http.Handler func ValidateHost(validHost func(host string) bool) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.ValidateHost") + defer span.End() + if !validHost(r.Host) { httputil.ErrorResponse(w, r, &httputil.Error{Code: http.StatusNotFound}) return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -145,13 +160,16 @@ func ValidateHost(validHost func(host string) bool) func(next http.Handler) http func Healthcheck(endpoint, msg string) func(http.Handler) http.Handler { f := func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.Healthcheck") + defer span.End() + if r.Method == "GET" && strings.EqualFold(r.URL.Path, endpoint) { w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) w.Write([]byte(msg)) return } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) } return http.HandlerFunc(fn) } diff --git a/internal/middleware/reverse_proxy.go b/internal/middleware/reverse_proxy.go index bd4e6e780..280caf873 100644 --- a/internal/middleware/reverse_proxy.go +++ b/internal/middleware/reverse_proxy.go @@ -6,11 +6,14 @@ import ( "github.com/pomerium/pomerium/internal/cryptutil" "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/telemetry/trace" ) func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.SignRequest") + defer span.End() jwt, err := signer.SignJWT( r.Header.Get(id), r.Header.Get(email), @@ -20,7 +23,7 @@ func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) f } else { r.Header.Set(header, jwt) } - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } @@ -29,6 +32,9 @@ func SignRequest(signer cryptutil.JWTSigner, id, email, groups, header string) f func StripPomeriumCookie(cookieName string) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := trace.StartSpan(r.Context(), "middleware.SignRequest") + defer span.End() + headers := make([]string, len(r.Cookies())) for _, cookie := range r.Cookies() { if cookie.Name != cookieName { @@ -36,7 +42,7 @@ func StripPomeriumCookie(cookieName string) func(next http.Handler) http.Handler } } r.Header.Set("Cookie", strings.Join(headers, ";")) - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) }) } } diff --git a/internal/telemetry/metrics/const.go b/internal/telemetry/metrics/const.go new file mode 100644 index 000000000..7066ec3c0 --- /dev/null +++ b/internal/telemetry/metrics/const.go @@ -0,0 +1,41 @@ +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" + +import ( + "go.opencensus.io/plugin/ocgrpc" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +// The following tags are applied to stats recorded by this package. +var ( + TagKeyHTTPMethod tag.Key = tag.MustNewKey("http_method") + TagKeyService tag.Key = tag.MustNewKey("service") + TagKeyGRPCService tag.Key = tag.MustNewKey("grpc_service") + TagKeyGRPCMethod tag.Key = tag.MustNewKey("grpc_method") + TagKeyHost tag.Key = tag.MustNewKey("host") + TagKeyDestination tag.Key = tag.MustNewKey("destination") +) + +// Default distributions used by views in this package. +var ( + DefaulHTTPSizeDistribution = view.Distribution( + 1, 256, 512, 1024, 2048, 8192, 16384, 32768, 65536, 131072, 262144, + 524288, 1048576, 2097152, 4194304, 8388608) + DefaultHTTPLatencyDistrubtion = view.Distribution( + 1, 2, 5, 7, 10, 25, 500, 750, 100, 250, 500, 750, 1000, 2500, 5000, + 7500, 10000, 25000, 50000, 75000, 100000) + grpcSizeDistribution = view.Distribution( + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, + 2048, 4096, 8192, 16384, + ) + DefaultMillisecondsDistribution = ocgrpc.DefaultMillisecondsDistribution +) + +// DefaultViews are a set of default views to view HTTP and GRPC metrics. +var ( + DefaultViews = [][]*view.View{ + GRPCServerViews, + HTTPServerViews, + GRPCClientViews, + GRPCServerViews} +) diff --git a/internal/metrics/interceptors.go b/internal/telemetry/metrics/grpc.go similarity index 68% rename from internal/metrics/interceptors.go rename to internal/telemetry/metrics/grpc.go index b736a7559..04c2b3d66 100644 --- a/internal/metrics/interceptors.go +++ b/internal/telemetry/metrics/grpc.go @@ -1,4 +1,4 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "context" @@ -12,93 +12,98 @@ import ( grpcstats "google.golang.org/grpc/stats" ) +// GRPC Views var ( - grpcSizeDistribution = view.Distribution( - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, - 2048, 4096, 8192, 16384, - ) - grcpLatencyDistribution = view.Distribution( - 1, 2, 5, 7, 10, 25, 50, 75, - 100, 250, 500, 750, 1000, - ) + // GRPCClientViews contains opencensus views for GRPC Client metrics. + GRPCClientViews = []*view.View{ + GRPCClientRequestCountView, + GRPCClientRequestDurationView, + GRPCClientResponseSizeView, + GRPCClientRequestSizeView} + // GRPCServerViews contains opencensus views for GRPC Server metrics. + GRPCServerViews = []*view.View{ + GRPCServerRequestCountView, + GRPCServerRequestDurationView, + GRPCServerResponseSizeView, + GRPCServerRequestSizeView} // GRPCServerRequestCountView is an OpenCensus view which counts GRPC Server // requests by pomerium service, grpc service, grpc method, and status GRPCServerRequestCountView = &view.View{ - Name: "grpc_server_requests_total", + Name: "grpc/server/requests_total", Measure: ocgrpc.ServerLatency, Description: "Total grpc Requests", - TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + TagKeys: []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService}, Aggregation: view.Count(), } // GRPCServerRequestDurationView is an OpenCensus view which tracks GRPC Server // request duration by pomerium service, grpc service, grpc method, and status GRPCServerRequestDurationView = &view.View{ - Name: "grpc_server_request_duration_ms", + Name: "grpc/server/request_duration_ms", Measure: ocgrpc.ServerLatency, Description: "grpc Request duration in ms", - TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, - Aggregation: grcpLatencyDistribution, + TagKeys: []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService}, + Aggregation: DefaultMillisecondsDistribution, } // GRPCServerResponseSizeView is an OpenCensus view which tracks GRPC Server // response size by pomerium service, grpc service, grpc method, and status GRPCServerResponseSizeView = &view.View{ - Name: "grpc_server_response_size_bytes", + Name: "grpc/server/response_size_bytes", Measure: ocgrpc.ServerSentBytesPerRPC, Description: "grpc Server Response Size in bytes", - TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + TagKeys: []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService}, Aggregation: grpcSizeDistribution, } // GRPCServerRequestSizeView is an OpenCensus view which tracks GRPC Server // request size by pomerium service, grpc service, grpc method, and status GRPCServerRequestSizeView = &view.View{ - Name: "grpc_server_request_size_bytes", + Name: "grpc/server/request_size_bytes", Measure: ocgrpc.ServerReceivedBytesPerRPC, Description: "grpc Server Request Size in bytes", - TagKeys: []tag.Key{keyService, keyGRPCMethod, ocgrpc.KeyServerStatus, keyGRPCService}, + TagKeys: []tag.Key{TagKeyService, TagKeyGRPCMethod, ocgrpc.KeyServerStatus, TagKeyGRPCService}, Aggregation: grpcSizeDistribution, } // GRPCClientRequestCountView is an OpenCensus view which tracks GRPC Client // requests by pomerium service, target host, grpc service, grpc method, and status GRPCClientRequestCountView = &view.View{ - Name: "grpc_client_requests_total", + Name: "grpc/client/requests_total", Measure: ocgrpc.ClientRoundtripLatency, Description: "Total grpc Client Requests", - TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus}, Aggregation: view.Count(), } // GRPCClientRequestDurationView is an OpenCensus view which tracks GRPC Client // request duration by pomerium service, target host, grpc service, grpc method, and status GRPCClientRequestDurationView = &view.View{ - Name: "grpc_client_request_duration_ms", + Name: "grpc/client/request_duration_ms", Measure: ocgrpc.ClientRoundtripLatency, Description: "grpc Client Request duration in ms", - TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, - Aggregation: grcpLatencyDistribution, + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus}, + Aggregation: DefaultMillisecondsDistribution, } // GRPCClientResponseSizeView is an OpenCensus view which tracks GRPC Client // response size by pomerium service, target host, grpc service, grpc method, and status GRPCClientResponseSizeView = &view.View{ - Name: "grpc_client_response_size_bytes", + Name: "grpc/client/response_size_bytes", Measure: ocgrpc.ClientReceivedBytesPerRPC, Description: "grpc Client Response Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus}, Aggregation: grpcSizeDistribution, } // GRPCClientRequestSizeView is an OpenCensus view which tracks GRPC Client // request size by pomerium service, target host, grpc service, grpc method, and status GRPCClientRequestSizeView = &view.View{ - Name: "grpc_client_request_size_bytes", + Name: "grpc/client/request_size_bytes", Measure: ocgrpc.ClientSentBytesPerRPC, Description: "grpc Client Request Size in bytes", - TagKeys: []tag.Key{keyService, keyHost, keyGRPCMethod, keyGRPCService, ocgrpc.KeyClientStatus}, + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyGRPCMethod, TagKeyGRPCService, ocgrpc.KeyClientStatus}, Aggregation: grpcSizeDistribution, } ) @@ -126,13 +131,13 @@ func GRPCClientInterceptor(service string) grpc.UnaryClientInterceptor { taggedCtx, tagErr := tag.New( ctx, - tag.Insert(keyService, service), - tag.Insert(keyHost, cc.Target()), - tag.Insert(keyGRPCMethod, rpcMethod), - tag.Insert(keyGRPCService, rpcService), + tag.Insert(TagKeyService, service), + tag.Insert(TagKeyHost, cc.Target()), + tag.Insert(TagKeyGRPCMethod, rpcMethod), + tag.Insert(TagKeyGRPCService, rpcService), ) if tagErr != nil { - log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/metrics: Failed to create context") + log.Warn().Err(tagErr).Str("context", "GRPCClientInterceptor").Msg("internal/telemetry: Failed to create context") return invoker(ctx, method, req, reply, cc, opts...) } @@ -165,12 +170,12 @@ func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats. taggedCtx, tagErr := tag.New( handledCtx, - tag.Insert(keyService, h.service), - tag.Insert(keyGRPCMethod, rpcMethod), - tag.Insert(keyGRPCService, rpcService), + tag.Insert(TagKeyService, h.service), + tag.Insert(TagKeyGRPCMethod, rpcMethod), + tag.Insert(TagKeyGRPCService, rpcService), ) if tagErr != nil { - log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/metrics: Failed to create context") + log.Warn().Err(tagErr).Str("context", "GRPCServerStatsHandler").Msg("internal/telemetry: Failed to create context") return handledCtx } @@ -180,6 +185,5 @@ func (h *GRPCServerStatsHandler) TagRPC(ctx context.Context, tagInfo *grpcstats. // NewGRPCServerStatsHandler creates a new GRPCServerStatsHandler for a pomerium service func NewGRPCServerStatsHandler(service string) grpcstats.Handler { - return &GRPCServerStatsHandler{service: service, Handler: &ocgrpc.ServerHandler{}} } diff --git a/internal/metrics/interceptors_test.go b/internal/telemetry/metrics/grpc_test.go similarity index 97% rename from internal/metrics/interceptors_test.go rename to internal/telemetry/metrics/grpc_test.go index 7a3470cc1..9dd8aa666 100644 --- a/internal/metrics/interceptors_test.go +++ b/internal/telemetry/metrics/grpc_test.go @@ -1,10 +1,11 @@ -package metrics +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "context" "testing" "go.opencensus.io/plugin/ocgrpc" + "go.opencensus.io/stats/view" "google.golang.org/grpc" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" @@ -97,8 +98,8 @@ func Test_GRPCClientInterceptor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - UnRegisterView(GRPCClientViews) - RegisterView(GRPCClientViews) + view.Unregister(GRPCClientViews...) + view.Register(GRPCClientViews...) invoker := testInvoker{ invokeResult: tt.errorCode, @@ -167,8 +168,8 @@ func Test_GRPCServerStatsHandler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - UnRegisterView(GRPCServerViews) - RegisterView(GRPCServerViews) + view.Unregister(GRPCServerViews...) + view.Register(GRPCServerViews...) statsHandler := NewGRPCServerStatsHandler("test_service") mockServerRPCHandle(statsHandler, tt.method, tt.errorCode) diff --git a/internal/metrics/helpers_test.go b/internal/telemetry/metrics/helpers_test.go similarity index 53% rename from internal/metrics/helpers_test.go rename to internal/telemetry/metrics/helpers_test.go index b8f02a2e7..96a7a2b50 100644 --- a/internal/metrics/helpers_test.go +++ b/internal/telemetry/metrics/helpers_test.go @@ -1,41 +1,12 @@ -package metrics +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( - "strings" "testing" "github.com/google/go-cmp/cmp" "go.opencensus.io/metric/metricdata" - "go.opencensus.io/stats/view" ) -func testDataRetrieval(v *view.View, t *testing.T, want string) { - if v == nil { - t.Fatalf("%s: nil view passed", t.Name()) - } - name := v.Name - data, err := view.RetrieveData(name) - - if err != nil { - t.Fatalf("%s: failed to retrieve data line %s", name, err) - } - - if want != "" && len(data) != 1 { - t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data)) - } - if want == "" && len(data) > 0 { - t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data)) - } else if want == "" { - return - } - - dataString := data[0].String() - - if want != "" && !strings.HasPrefix(dataString, want) { - t.Errorf("%s: Found unexpected data row: \nwant: %s\ngot: %s\n", name, want, dataString) - } -} - func testMetricRetrieval(metrics []*metricdata.Metric, t *testing.T, labels []metricdata.LabelValue, value interface{}, name string) { switch value.(type) { case int64: diff --git a/internal/telemetry/metrics/http.go b/internal/telemetry/metrics/http.go new file mode 100644 index 000000000..9a2586208 --- /dev/null +++ b/internal/telemetry/metrics/http.go @@ -0,0 +1,157 @@ +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" + +import ( + "fmt" + "net/http" + + "github.com/pomerium/pomerium/internal/log" + "github.com/pomerium/pomerium/internal/tripper" + "go.opencensus.io/plugin/ochttp" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +// HTTP Views +var ( + // HTTPClientViews contains opencensus views for HTTP Client metrics. + HTTPClientViews = []*view.View{ + HTTPClientRequestCountView, + HTTPClientRequestDurationView, + HTTPClientResponseSizeView} + // HTTPServerViews contains opencensus views for HTTP Server metrics. + HTTPServerViews = []*view.View{ + HTTPServerRequestCountView, + HTTPServerRequestDurationView, + HTTPServerRequestSizeView, + HTTPServerResponseSizeView} + + // HTTPServerRequestCountView is an OpenCensus View that tracks HTTP server + // requests by pomerium service, host, method and status + HTTPServerRequestCountView = &view.View{ + Name: "http/server/requests_total", + Measure: ochttp.ServerLatency, + Description: "Total HTTP Requests", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode}, + Aggregation: view.Count(), + } + + // HTTPServerRequestDurationView is an OpenCensus view that tracks HTTP + // server request duration by pomerium service, host, method and status + HTTPServerRequestDurationView = &view.View{ + Name: "http/server/request_duration_ms", + Measure: ochttp.ServerLatency, + Description: "HTTP Request duration in ms", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode}, + Aggregation: DefaultHTTPLatencyDistrubtion, + } + + // HTTPServerRequestSizeView is an OpenCensus view that tracks HTTP server + // request size by pomerium service, host and method + HTTPServerRequestSizeView = &view.View{ + Name: "http/server/request_size_bytes", + Measure: ochttp.ServerRequestBytes, + Description: "HTTP Server Request Size in bytes", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod}, + Aggregation: DefaulHTTPSizeDistribution, + } + + // HTTPServerResponseSizeView is an OpenCensus view that tracks HTTP server + // response size by pomerium service, host, method and status + HTTPServerResponseSizeView = &view.View{ + Name: "http/server/response_size_bytes", + Measure: ochttp.ServerResponseBytes, + Description: "HTTP Server Response Size in bytes", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode}, + Aggregation: DefaulHTTPSizeDistribution, + } + + // HTTPClientRequestCountView is an OpenCensus View that tracks HTTP client + // requests by pomerium service, destination, host, method and status + HTTPClientRequestCountView = &view.View{ + Name: "http/client/requests_total", + Measure: ochttp.ClientRoundtripLatency, + Description: "Total HTTP Client Requests", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination}, + Aggregation: view.Count(), + } + + // HTTPClientRequestDurationView is an OpenCensus view that tracks HTTP + // client request duration by pomerium service, destination, host, method and status + HTTPClientRequestDurationView = &view.View{ + Name: "http/client/request_duration_ms", + Measure: ochttp.ClientRoundtripLatency, + Description: "HTTP Client Request duration in ms", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination}, + Aggregation: DefaultHTTPLatencyDistrubtion, + } + + // HTTPClientResponseSizeView is an OpenCensus view that tracks HTTP client + // esponse size by pomerium service, destination, host, method and status + HTTPClientResponseSizeView = &view.View{ + Name: "http/client/response_size_bytes", + Measure: ochttp.ClientReceivedBytes, + Description: "HTTP Client Response Size in bytes", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, ochttp.StatusCode, TagKeyDestination}, + Aggregation: DefaulHTTPSizeDistribution, + } + + // HTTPClientRequestSizeView is an OpenCensus view that tracks HTTP client + //request size by pomerium service, destination, host and method + HTTPClientRequestSizeView = &view.View{ + Name: "http/client/response_size_bytes", + Measure: ochttp.ClientSentBytes, + Description: "HTTP Client Response Size in bytes", + TagKeys: []tag.Key{TagKeyService, TagKeyHost, TagKeyHTTPMethod, TagKeyDestination}, + Aggregation: DefaulHTTPSizeDistribution, + } +) + +// HTTPMetricsHandler creates a metrics middleware for incoming HTTP requests +func HTTPMetricsHandler(service string) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, tagErr := tag.New( + r.Context(), + tag.Insert(TagKeyService, service), + tag.Insert(TagKeyHost, r.Host), + tag.Insert(TagKeyHTTPMethod, r.Method), + ) + if tagErr != nil { + log.Warn().Err(tagErr).Str("context", "HTTPMetricsHandler"). + Msg("telemetry/metrics: failed to create metrics tag") + next.ServeHTTP(w, r) + return + } + + ocHandler := ochttp.Handler{ + Handler: next, + FormatSpanName: func(r *http.Request) string { + return fmt.Sprintf("%s%s", r.Host, r.URL.Path) + }, + } + ocHandler.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} + +// HTTPMetricsRoundTripper creates a metrics tracking tripper for outbound HTTP Requests +func HTTPMetricsRoundTripper(service string, destination string) func(next http.RoundTripper) http.RoundTripper { + return func(next http.RoundTripper) http.RoundTripper { + return tripper.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + ctx, tagErr := tag.New( + r.Context(), + tag.Insert(TagKeyService, service), + tag.Insert(TagKeyHost, r.Host), + tag.Insert(TagKeyHTTPMethod, r.Method), + tag.Insert(TagKeyDestination, destination), + ) + if tagErr != nil { + log.Warn().Err(tagErr).Str("context", "HTTPMetricsRoundTripper").Msg("telemetry/metrics: failed to create metrics tag") + return next.RoundTrip(r) + } + + ocTransport := ochttp.Transport{Base: next} + return ocTransport.RoundTrip(r.WithContext(ctx)) + }) + } +} diff --git a/internal/metrics/middleware_test.go b/internal/telemetry/metrics/http_test.go similarity index 91% rename from internal/metrics/middleware_test.go rename to internal/telemetry/metrics/http_test.go index e13e9eca9..050b74517 100644 --- a/internal/metrics/middleware_test.go +++ b/internal/telemetry/metrics/http_test.go @@ -1,4 +1,4 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "bytes" @@ -7,13 +7,40 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "strings" "testing" - "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/tripper" "go.opencensus.io/stats/view" ) +func testDataRetrieval(v *view.View, t *testing.T, want string) { + if v == nil { + t.Fatalf("%s: nil view passed", t.Name()) + } + name := v.Name + data, err := view.RetrieveData(name) + + if err != nil { + t.Fatalf("%s: failed to retrieve data line %s", name, err) + } + + if want != "" && len(data) != 1 { + t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data)) + } + if want == "" && len(data) > 0 { + t.Fatalf("%s: received incorrect number of data rows: %d", name, len(data)) + } else if want == "" { + return + } + + dataString := data[0].String() + + if want != "" && !strings.HasPrefix(dataString, want) { + t.Errorf("%s: Found unexpected data row: \nwant: %s\ngot: %s\n", name, want, dataString) + } +} + func newTestMux() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/good", func(w http.ResponseWriter, r *http.Request) { @@ -25,10 +52,6 @@ func newTestMux() http.Handler { func Test_HTTPMetricsHandler(t *testing.T) { - chain := middleware.NewChain() - chain = chain.Append(HTTPMetricsHandler("test_service")) - chainHandler := chain.Then(newTestMux()) - tests := []struct { name string url string @@ -73,7 +96,9 @@ func Test_HTTPMetricsHandler(t *testing.T) { req := httptest.NewRequest(tt.verb, tt.url, new(bytes.Buffer)) rec := httptest.NewRecorder() - chainHandler.ServeHTTP(rec, req) + + h := HTTPMetricsHandler("test_service")(newTestMux()) + h.ServeHTTP(rec, req) testDataRetrieval(HTTPServerRequestSizeView, t, tt.wanthttpServerRequestSize) testDataRetrieval(HTTPServerResponseSizeView, t, tt.wanthttpServerResponseSize) diff --git a/internal/metrics/info.go b/internal/telemetry/metrics/info.go similarity index 75% rename from internal/metrics/info.go rename to internal/telemetry/metrics/info.go index d822e7482..ec1f8f151 100644 --- a/internal/metrics/info.go +++ b/internal/telemetry/metrics/info.go @@ -1,4 +1,4 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "context" @@ -8,6 +8,7 @@ import ( "github.com/pomerium/pomerium/internal/log" "github.com/pomerium/pomerium/internal/version" + "go.opencensus.io/metric" "go.opencensus.io/metric/metricdata" "go.opencensus.io/metric/metricproducer" @@ -17,44 +18,53 @@ import ( ) var ( - //buildInfo = stats.Int64("build_info", "Build Metadata", "1") - configLastReload = stats.Int64("config_last_reload_success_timestamp", "Timestamp of last successful config reload", "seconds") - configLastReloadSuccess = stats.Int64("config_last_reload_success", "Returns 1 if last reload was successful", "1") - registry = newMetricRegistry() + // InfoViews contains opencensus views for informational metrics about + // pomerium itself. + InfoViews = []*view.View{ConfigLastReloadView, ConfigLastReloadSuccessView} + + configLastReload = stats.Int64( + "config_last_reload_success_timestamp", + "Timestamp of last successful config reload", + "seconds") + configLastReloadSuccess = stats.Int64( + "config_last_reload_success", + "Returns 1 if last reload was successful", + "1") + registry = newMetricRegistry() // ConfigLastReloadView contains the timestamp the configuration was last - // reloaded, labeled by service + // reloaded, labeled by service. ConfigLastReloadView = &view.View{ Name: configLastReload.Name(), Description: configLastReload.Description(), Measure: configLastReload, - TagKeys: []tag.Key{keyService}, + TagKeys: []tag.Key{TagKeyService}, Aggregation: view.LastValue(), } // ConfigLastReloadSuccessView contains the result of the last configuration - // reload, labeled by service + // reload, labeled by service. ConfigLastReloadSuccessView = &view.View{ Name: configLastReloadSuccess.Name(), Description: configLastReloadSuccess.Description(), Measure: configLastReloadSuccess, - TagKeys: []tag.Key{keyService}, + TagKeys: []tag.Key{TagKeyService}, Aggregation: view.LastValue(), } ) -// SetConfigInfo records the status, checksum and timestamp of a configuration reload. You must register InfoViews or the related -// config views before calling +// SetConfigInfo records the status, checksum and timestamp of a configuration +// reload. You must register InfoViews or the related config views before calling func SetConfigInfo(service string, success bool, checksum string) { if success { - serviceTag := tag.Insert(keyService, service) + serviceTag := tag.Insert(TagKeyService, service) if err := stats.RecordWithTags( context.Background(), []tag.Mutator{serviceTag}, configLastReload.M(time.Now().Unix()), ); err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to record config checksum timestamp") + log.Error().Err(err).Msg("internal/telemetry: failed to record config checksum timestamp") } if err := stats.RecordWithTags( @@ -62,7 +72,7 @@ func SetConfigInfo(service string, success bool, checksum string) { []tag.Mutator{serviceTag}, configLastReloadSuccess.M(1), ); err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to record config reload") + log.Error().Err(err).Msg("internal/telemetry: failed to record config reload") } } else { stats.Record(context.Background(), configLastReloadSuccess.M(0)) @@ -96,7 +106,7 @@ func (r *metricRegistry) init() { metric.WithLabelKeys("service", "version", "revision", "goversion"), ) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to register build info metric") + log.Error().Err(err).Msg("internal/telemetry: failed to register build info metric") } r.configChecksum, err = r.registry.AddFloat64Gauge("config_checksum_decimal", @@ -104,7 +114,7 @@ func (r *metricRegistry) init() { metric.WithLabelKeys("service"), ) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to register config checksum metric") + log.Error().Err(err).Msg("internal/telemetry: failed to register config checksum metric") } r.policyCount, err = r.registry.AddInt64DerivedGauge("policy_count_total", @@ -112,7 +122,7 @@ func (r *metricRegistry) init() { metric.WithLabelKeys("service"), ) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to register policy count metric") + log.Error().Err(err).Msg("internal/telemetry: failed to register policy count metric") } }) } @@ -130,7 +140,7 @@ func (r *metricRegistry) setBuildInfo(service string) { metricdata.NewLabelValue((runtime.Version())), ) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to get build info metric") + log.Error().Err(err).Msg("internal/telemetry: failed to get build info metric") } // This sets our build_info metric to a constant 1 per @@ -155,7 +165,7 @@ func (r *metricRegistry) setConfigChecksum(service string, checksum uint64) { } m, err := r.configChecksum.GetEntry(metricdata.NewLabelValue(service)) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to get config checksum metric") + log.Error().Err(err).Msg("internal/telemetry: failed to get config checksum metric") } m.Set(float64(checksum)) } @@ -172,7 +182,7 @@ func (r *metricRegistry) addPolicyCountCallback(service string, f func() int64) } err := r.policyCount.UpsertEntry(f, metricdata.NewLabelValue(service)) if err != nil { - log.Error().Err(err).Msg("internal/metrics: failed to get policy count metric") + log.Error().Err(err).Msg("internal/telemetry: failed to get policy count metric") } } diff --git a/internal/metrics/info_test.go b/internal/telemetry/metrics/info_test.go similarity index 95% rename from internal/metrics/info_test.go rename to internal/telemetry/metrics/info_test.go index c29285a49..efce2d0a4 100644 --- a/internal/metrics/info_test.go +++ b/internal/telemetry/metrics/info_test.go @@ -1,4 +1,4 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "runtime" @@ -8,6 +8,7 @@ import ( "go.opencensus.io/metric/metricdata" "go.opencensus.io/metric/metricproducer" + "go.opencensus.io/stats/view" ) func Test_SetConfigInfo(t *testing.T) { @@ -24,9 +25,8 @@ func Test_SetConfigInfo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - UnRegisterView(InfoViews) - RegisterView(InfoViews) - + view.Unregister(InfoViews...) + view.Register(InfoViews...) SetConfigInfo("test_service", tt.success, tt.checksum) testDataRetrieval(ConfigLastReloadView, t, tt.wantLastReload) diff --git a/internal/telemetry/metrics/providers.go b/internal/telemetry/metrics/providers.go new file mode 100644 index 000000000..c5ed505df --- /dev/null +++ b/internal/telemetry/metrics/providers.go @@ -0,0 +1,39 @@ +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" + +import ( + "fmt" + "net/http" + + ocprom "contrib.go.opencensus.io/exporter/prometheus" + prom "github.com/prometheus/client_golang/prometheus" + "go.opencensus.io/stats/view" +) + +// PrometheusHandler creates an exporter that exports stats to Prometheus +// and returns a handler suitable for exporting metrics. +func PrometheusHandler() (http.Handler, error) { + if err := registerDefaultViews(); err != nil { + return nil, fmt.Errorf("internal/telemetry: failed registering views") + } + reg := prom.DefaultRegisterer.(*prom.Registry) + exporter, err := ocprom.NewExporter( + ocprom.Options{ + Namespace: "pomerium", + Registry: reg, + }) + if err != nil { + return nil, fmt.Errorf("internal/telemetry: prometheus exporter: %v", err) + } + view.RegisterExporter(exporter) + mux := http.NewServeMux() + mux.Handle("/metrics", exporter) + return mux, nil +} + +func registerDefaultViews() error { + var views []*view.View + for _, v := range DefaultViews { + views = append(views, v...) + } + return view.Register(views...) +} diff --git a/internal/metrics/exporter_test.go b/internal/telemetry/metrics/providers_test.go similarity index 81% rename from internal/metrics/exporter_test.go rename to internal/telemetry/metrics/providers_test.go index 2d9b1566f..6c5e6ea1c 100644 --- a/internal/metrics/exporter_test.go +++ b/internal/telemetry/metrics/providers_test.go @@ -1,4 +1,4 @@ -package metrics // import "github.com/pomerium/pomerium/internal/metrics" +package metrics // import "github.com/pomerium/pomerium/internal/telemetry/metrics" import ( "bytes" @@ -8,9 +8,11 @@ import ( "testing" ) -func Test_newPromHTTPHandler(t *testing.T) { - h := newPromHTTPHandler() - +func Test_PrometheusHandler(t *testing.T) { + h, err := PrometheusHandler() + if err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer)) rec := httptest.NewRecorder() h.ServeHTTP(rec, req) diff --git a/internal/telemetry/trace/trace.go b/internal/telemetry/trace/trace.go new file mode 100644 index 000000000..d52a5fa79 --- /dev/null +++ b/internal/telemetry/trace/trace.go @@ -0,0 +1,74 @@ +package trace // import "github.com/pomerium/pomerium/internal/telemetry/trace" + +import ( + "context" + "fmt" + + "github.com/pomerium/pomerium/internal/log" + + "contrib.go.opencensus.io/exporter/jaeger" + "go.opencensus.io/trace" +) + +const ( + JaegerTracingProviderName = "jaeger" +) + +// TracingOptions contains the configurations settings for a http server. +type TracingOptions struct { + // Shared + Provider string + Service string + Debug bool + + // Jaeger + + // CollectorEndpoint is the full url to the Jaeger HTTP Thrift collector. + // For example, http://localhost:14268/api/traces + JaegerCollectorEndpoint string `mapstructure:"tracing_jaeger_collector_endpoint"` + // AgentEndpoint instructs exporter to send spans to jaeger-agent at this address. + // For example, localhost:6831. + JaegerAgentEndpoint string `mapstructure:"tracing_jaeger_agent_endpoint"` +} + +func RegisterTracing(opts *TracingOptions) error { + var err error + switch opts.Provider { + case JaegerTracingProviderName: + err = registerJaeger(opts) + default: + return fmt.Errorf("telemetry/trace: provider %s unknown", opts.Provider) + } + if err != nil { + return err + } + if opts.Debug { + log.Debug().Msg("telemetry/trace: debug on, sample everything") + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + } + log.Debug().Interface("Opts", opts).Msg("telemetry/trace: exporter created") + return nil +} + +func registerJaeger(opts *TracingOptions) error { + jex, err := jaeger.NewExporter( + jaeger.Options{ + AgentEndpoint: opts.JaegerAgentEndpoint, + CollectorEndpoint: opts.JaegerCollectorEndpoint, + ServiceName: opts.Service, + }) + if err != nil { + return err + } + trace.RegisterExporter(jex) + return nil +} + +// StartSpan starts a new child span of the current span in the context. If +// there is no span in the context, creates a new trace and span. +// +// Returned context contains the newly created span. You can use it to +// propagate the returned span in process. +func StartSpan(ctx context.Context, name string, o ...trace.StartOption) (context.Context, *trace.Span) { + return trace.StartSpan(ctx, name, o...) +} diff --git a/internal/telemetry/trace/trace_test.go b/internal/telemetry/trace/trace_test.go new file mode 100644 index 000000000..745060104 --- /dev/null +++ b/internal/telemetry/trace/trace_test.go @@ -0,0 +1,23 @@ +package trace // import "github.com/pomerium/pomerium/internal/telemetry/trace" + +import "testing" + +func TestRegisterTracing(t *testing.T) { + tests := []struct { + name string + opts *TracingOptions + wantErr bool + }{ + {"jaeger", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger"}, false}, + {"jaeger with debug", &TracingOptions{JaegerAgentEndpoint: "localhost:6831", Service: "all", Provider: "jaeger", Debug: true}, false}, + {"jaeger no endpoint", &TracingOptions{JaegerAgentEndpoint: "", Service: "all", Provider: "jaeger"}, true}, + {"unknown provider", &TracingOptions{JaegerAgentEndpoint: "localhost:0", Service: "all", Provider: "Lucius Cornelius Sulla"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := RegisterTracing(tt.opts); (err != nil) != tt.wantErr { + t.Errorf("RegisterTracing() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/proxy/clients/authenticate_client.go b/proxy/clients/authenticate_client.go index a807b792a..572828526 100644 --- a/proxy/clients/authenticate_client.go +++ b/proxy/clients/authenticate_client.go @@ -3,12 +3,12 @@ package clients // import "github.com/pomerium/pomerium/proxy/clients" import ( "context" "errors" - "time" - - "google.golang.org/grpc" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" pb "github.com/pomerium/pomerium/proto/authenticate" + + "google.golang.org/grpc" ) // Authenticator provides the authenticate service interface @@ -48,11 +48,12 @@ type AuthenticateGRPC struct { // Redeem makes an RPC call to the authenticate service to creates a session state // from an encrypted code provided as a result of an oauth2 callback process. func (a *AuthenticateGRPC) Redeem(ctx context.Context, code string) (*sessions.SessionState, error) { + ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Redeem") + defer span.End() + if code == "" { return nil, errors.New("missing code") } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() protoSession, err := a.client.Authenticate(ctx, &pb.AuthenticateRequest{Code: code}) if err != nil { return nil, err @@ -68,6 +69,9 @@ func (a *AuthenticateGRPC) Redeem(ctx context.Context, code string) (*sessions.S // user's session. Requires a valid refresh token. Will return an error if the identity provider // has revoked the session or if the refresh token is no longer valid in this context. func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState) (*sessions.SessionState, error) { + ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Refresh") + defer span.End() + if s.RefreshToken == "" { return nil, errors.New("missing refresh token") } @@ -75,14 +79,7 @@ func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState if err != nil { return nil, err } - // todo(bdd): handle request id in grpc receiver and add to ctx logger - // reqID, ok := middleware.IDFromCtx(ctx) - // if ok { - // md := metadata.Pairs("req_id", reqID) - // ctx = metadata.NewOutgoingContext(ctx, md) - // } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() + // todo(bdd): add grpc specific timeouts to main options // todo(bdd): handle request id (metadata!?) in grpc receiver and add to ctx logger reply, err := a.client.Refresh(ctx, req) @@ -100,18 +97,13 @@ func (a *AuthenticateGRPC) Refresh(ctx context.Context, s *sessions.SessionState // does NOT do nonce or revokation validation. // https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation func (a *AuthenticateGRPC) Validate(ctx context.Context, idToken string) (bool, error) { + ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Validate") + defer span.End() + if idToken == "" { return false, errors.New("missing id token") } - // todo(bdd): add grpc specific timeouts to main options - // todo(bdd): handle request id in grpc receiver and add to ctx logger - // reqID, ok := middleware.IDFromCtx(ctx) - // if ok { - // md := metadata.Pairs("req_id", reqID) - // ctx = metadata.NewOutgoingContext(ctx, md) - // } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() + r, err := a.client.Validate(ctx, &pb.ValidateRequest{IdToken: idToken}) if err != nil { return false, err diff --git a/proxy/clients/authorize_client.go b/proxy/clients/authorize_client.go index e444f739f..229fa87b5 100644 --- a/proxy/clients/authorize_client.go +++ b/proxy/clients/authorize_client.go @@ -3,12 +3,12 @@ package clients // import "github.com/pomerium/pomerium/proxy/clients" import ( "context" "errors" - "time" - - "google.golang.org/grpc" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/trace" pb "github.com/pomerium/pomerium/proto/authorize" + + "google.golang.org/grpc" ) // Authorizer provides the authorize service interface @@ -47,11 +47,12 @@ type AuthorizeGRPC struct { // Authorize takes a route and user session and returns whether the // request is valid per access policy func (a *AuthorizeGRPC) Authorize(ctx context.Context, route string, s *sessions.SessionState) (bool, error) { + ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.Authorize") + defer span.End() + if s == nil { return false, errors.New("session cannot be nil") } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() response, err := a.client.Authorize(ctx, &pb.Identity{ Route: route, User: s.User, @@ -65,11 +66,12 @@ func (a *AuthorizeGRPC) Authorize(ctx context.Context, route string, s *sessions // IsAdmin takes a session and returns whether the user is an administrator func (a *AuthorizeGRPC) IsAdmin(ctx context.Context, s *sessions.SessionState) (bool, error) { + ctx, span := trace.StartSpan(ctx, "proxy.client.grpc.IsAdmin") + defer span.End() + if s == nil { return false, errors.New("session cannot be nil") } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() response, err := a.client.IsAdmin(ctx, &pb.Identity{Email: s.Email, Groups: s.Groups}) return response.GetIsAdmin(), err } diff --git a/proxy/clients/clients.go b/proxy/clients/clients.go index 8f6d234fa..794e4bd08 100644 --- a/proxy/clients/clients.go +++ b/proxy/clients/clients.go @@ -11,8 +11,9 @@ import ( "strings" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/metrics" "github.com/pomerium/pomerium/internal/middleware" + "github.com/pomerium/pomerium/internal/telemetry/metrics" + "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/proxy/proxy.go b/proxy/proxy.go index a4690c9b1..a7ba6c80f 100755 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -8,6 +8,7 @@ import ( "fmt" "html/template" stdlog "log" + "net" "net/http" "net/http/httputil" "net/url" @@ -16,9 +17,10 @@ import ( "github.com/pomerium/pomerium/internal/config" "github.com/pomerium/pomerium/internal/cryptutil" "github.com/pomerium/pomerium/internal/log" - "github.com/pomerium/pomerium/internal/metrics" "github.com/pomerium/pomerium/internal/middleware" "github.com/pomerium/pomerium/internal/sessions" + "github.com/pomerium/pomerium/internal/telemetry/metrics" + "github.com/pomerium/pomerium/internal/telemetry/trace" "github.com/pomerium/pomerium/internal/templates" "github.com/pomerium/pomerium/internal/tripper" "github.com/pomerium/pomerium/proxy/clients" @@ -196,11 +198,19 @@ func (p *Proxy) UpdatePolicies(opts *config.Options) error { } proxy := NewReverseProxy(policy.Destination) // build http transport (roundtripper) middleware chain - // todo(bdd): this will make vet complain, it is safe - // and can be replaced with transport.Clone() in go 1.13 - // https://go-review.googlesource.com/c/go/+/174597/ - // https://github.com/golang/go/issues/26013#issuecomment-399481302 - transport := *(http.DefaultTransport.(*http.Transport)) + // todo(bdd): replace with transport.Clone() in go 1.13 + transport := http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } c := tripper.NewChain() c = c.Append(metrics.HTTPMetricsRoundTripper("proxy", policy.Destination.Host)) if policy.TLSSkipVerify { @@ -236,7 +246,9 @@ type UpstreamProxy struct { // ServeHTTP handles the second (reverse-proxying) leg of pomerium's request flow func (u *UpstreamProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - u.handler.ServeHTTP(w, r) + ctx, span := trace.StartSpan(r.Context(), fmt.Sprintf("%s%s", r.Host, r.URL.Path)) + defer span.End() + u.handler.ServeHTTP(w, r.WithContext(ctx)) } // NewReverseProxy returns a new ReverseProxy that routes URLs to the scheme, host, and