core: more metrics (#5629)

## Summary
Add some more metrics:

- Authenticate token verification
- Authorization log duration
- Authorization evaluator and header evaluator
- IDP token session creator

HTTP and gRPC endpoints are already instrumented via middleware, which
covers authenticate, proxy and databroker endpoints. Postgres is also
already instrumented using `otelpgx`.

## Related issues
-
[ENG-2407](https://linear.app/pomerium/issue/ENG-2407/add-additional-metrics-and-tracing-spans-to-pomerium)


## Checklist

- [x] reference any related issues
- [ ] updated unit tests
- [ ] add appropriate label (`enhancement`, `bug`, `breaking`,
`dependencies`, `ci`)
- [x] ready for review

authorize/authorize.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/metric"
 	oteltrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
@@ -24,6 +25,8 @@ import (
 
 // Authorize struct holds
 type Authorize struct {
+	logDuration metric.Int64Histogram
+
 	state         *atomicutil.Value[*authorizeState]
 	store         *store.Store
 	currentConfig *atomicutil.Value[*config.Config]
@@ -39,6 +42,10 @@ func New(ctx context.Context, cfg *config.Config) (*Authorize, error) {
 	tracer := tracerProvider.Tracer(trace.PomeriumCoreTracer)
 
 	a := &Authorize{
+		logDuration: metrics.Int64Histogram("authorize.log.duration",
+			metric.WithDescription("Duration of authorize log execution."),
+			metric.WithUnit("ms")),
+
 		currentConfig:  atomicutil.NewValue(cfg),
 		store:          store.New(),
 		tracerProvider: tracerProvider,

authorize/evaluator/evaluator.go

@@ -15,6 +15,7 @@ import (
 	"github.com/go-jose/go-jose/v3"
 	"github.com/hashicorp/go-set/v3"
 	"github.com/open-policy-agent/opa/rego"
+	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/types/known/structpb"
 
@@ -24,6 +25,7 @@ import (
 	"github.com/pomerium/pomerium/internal/errgrouputil"
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/pkg/contextutil"
 	"github.com/pomerium/pomerium/pkg/cryptutil"
 	"github.com/pomerium/pomerium/pkg/policy/criteria"
@@ -142,6 +144,9 @@ type Result struct {
 
 // An Evaluator evaluates policies.
 type Evaluator struct {
+	evaluationCount, allowCount, denyCount metric.Int64Counter
+	evaluationDuration                     metric.Int64Histogram
+
 	store                 *store.Store
 	policyEvaluators      map[string]*PolicyEvaluator
 	headersEvaluators     *HeadersEvaluator
@@ -164,6 +169,19 @@ func New(
 	}
 
 	e := &Evaluator{
+		evaluationCount: metrics.Int64Counter("authorize.evaluator.evaluations",
+			metric.WithDescription("Number of evaluations."),
+			metric.WithUnit("{evaluation}")),
+		allowCount: metrics.Int64Counter("authorize.evaluator.allowals",
+			metric.WithDescription("Number of allowals."),
+			metric.WithUnit("{allowal}")),
+		denyCount: metrics.Int64Counter("authorize.evaluator.denials",
+			metric.WithDescription("Number of denials."),
+			metric.WithUnit("{denial}")),
+		evaluationDuration: metrics.Int64Histogram("authorize.evaluator.evaluation.duration",
+			metric.WithDescription("Duration of evaluation."),
+			metric.WithUnit("ms")),
+
 		store:                 store,
 		clientCA:              cfg.ClientCA,
 		clientCRL:             cfg.ClientCRL,
@@ -252,6 +270,8 @@ func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error)
 	ctx, span := trace.Continue(ctx, "authorize.Evaluator.Evaluate")
 	defer span.End()
 
+	start := time.Now()
+
 	eg, ctx := errgroup.WithContext(ctx)
 
 	var policyOutput *PolicyResponse
@@ -277,6 +297,14 @@ func (e *Evaluator) Evaluate(ctx context.Context, req *Request) (*Result, error)
 		return nil, err
 	}
 
+	e.evaluationCount.Add(ctx, 1)
+	if policyOutput.Deny.Value {
+		e.denyCount.Add(ctx, 1)
+	} else if policyOutput.Allow.Value {
+		e.allowCount.Add(ctx, 1)
+	}
+	e.evaluationDuration.Record(ctx, time.Since(start).Microseconds())
+
 	res := &Result{
 		Allow:               policyOutput.Allow,
 		Deny:                policyOutput.Deny,

authorize/evaluator/headers_evaluator.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/open-policy-agent/opa/rego"
+	"go.opentelemetry.io/otel/metric"
 
 	"github.com/pomerium/pomerium/authorize/internal/store"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/pkg/telemetry/trace"
 )
 
@@ -20,12 +22,22 @@ type HeadersResponse struct {
 
 // A HeadersEvaluator evaluates the headers.rego script.
 type HeadersEvaluator struct {
+	evaluationCount    metric.Int64Counter
+	evaluationDuration metric.Int64Histogram
+
 	store *store.Store
 }
 
 // NewHeadersEvaluator creates a new HeadersEvaluator.
 func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator {
 	return &HeadersEvaluator{
+		evaluationCount: metrics.Int64Counter("authorize.header_evaluator.evaluations",
+			metric.WithDescription("Number of header evaluations."),
+			metric.WithUnit("{evaluation}")),
+		evaluationDuration: metrics.Int64Histogram("authorize.header_evaluator.evaluation.duration",
+			metric.WithDescription("Duration of header evaluation."),
+			metric.WithUnit("ms")),
+
 		store: store,
 	}
 }
@@ -35,6 +47,9 @@ func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options .
 	ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate")
 	defer span.End()
 
+	e.evaluationCount.Add(ctx, 1)
+	start := time.Now()
+
 	ectx := new(rego.EvalContext)
 	for _, option := range options {
 		option(ectx)
@@ -43,5 +58,7 @@ func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options .
 	if now.IsZero() {
 		now = time.Now()
 	}
-	return newHeadersEvaluatorEvaluation(e, req, now).execute(ctx)
+	res, err := newHeadersEvaluatorEvaluation(e, req, now).execute(ctx)
+	e.evaluationDuration.Record(ctx, time.Since(start).Milliseconds())
+	return res, err
 }

authorize/log.go

@@ -2,6 +2,7 @@ package authorize
 
 import (
 	"context"
+	"time"
 
 	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/rs/zerolog"
@@ -25,6 +26,8 @@ func (a *Authorize) logAuthorizeCheck(
 	ctx, span := a.tracer.Start(ctx, "authorize.grpc.LogAuthorizeCheck")
 	defer span.End()
 
+	start := time.Now()
+
 	hdrs := req.HTTP.Headers
 	impersonateDetails := a.getImpersonateDetails(ctx, s)
 
@@ -59,6 +62,7 @@ func (a *Authorize) logAuthorizeCheck(
 	}
 
 	evt.Msg("authorize check")
+	a.logDuration.Record(ctx, time.Since(start).Milliseconds())
 }
 
 type impersonateDetails struct {