core: more metrics (#5629)

## Summary
Add some more metrics:

- Authenticate token verification
- Authorization log duration
- Authorization evaluator and header evaluator
- IDP token session creator

HTTP and gRPC endpoints are already instrumented via middleware, which
covers authenticate, proxy and databroker endpoints. Postgres is also
already instrumented using `otelpgx`.

## Related issues
-
[ENG-2407](https://linear.app/pomerium/issue/ENG-2407/add-additional-metrics-and-tracing-spans-to-pomerium)


## Checklist

- [x] reference any related issues
- [ ] updated unit tests
- [ ] add appropriate label (`enhancement`, `bug`, `breaking`,
`dependencies`, `ci`)
- [x] ready for review

authorize/evaluator/headers_evaluator.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/open-policy-agent/opa/rego"
+	"go.opentelemetry.io/otel/metric"
 
 	"github.com/pomerium/pomerium/authorize/internal/store"
 	"github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 	"github.com/pomerium/pomerium/pkg/telemetry/trace"
 )
 
@@ -20,12 +22,22 @@ type HeadersResponse struct {
 
 // A HeadersEvaluator evaluates the headers.rego script.
 type HeadersEvaluator struct {
+	evaluationCount    metric.Int64Counter
+	evaluationDuration metric.Int64Histogram
+
 	store *store.Store
 }
 
 // NewHeadersEvaluator creates a new HeadersEvaluator.
 func NewHeadersEvaluator(store *store.Store) *HeadersEvaluator {
 	return &HeadersEvaluator{
+		evaluationCount: metrics.Int64Counter("authorize.header_evaluator.evaluations",
+			metric.WithDescription("Number of header evaluations."),
+			metric.WithUnit("{evaluation}")),
+		evaluationDuration: metrics.Int64Histogram("authorize.header_evaluator.evaluation.duration",
+			metric.WithDescription("Duration of header evaluation."),
+			metric.WithUnit("ms")),
+
 		store: store,
 	}
 }
@@ -35,6 +47,9 @@ func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options .
 	ctx, span := trace.Continue(ctx, "authorize.HeadersEvaluator.Evaluate")
 	defer span.End()
 
+	e.evaluationCount.Add(ctx, 1)
+	start := time.Now()
+
 	ectx := new(rego.EvalContext)
 	for _, option := range options {
 		option(ectx)
@@ -43,5 +58,7 @@ func (e *HeadersEvaluator) Evaluate(ctx context.Context, req *Request, options .
 	if now.IsZero() {
 		now = time.Now()
 	}
-	return newHeadersEvaluatorEvaluation(e, req, now).execute(ctx)
+	res, err := newHeadersEvaluatorEvaluation(e, req, now).execute(ctx)
+	e.evaluationDuration.Record(ctx, time.Since(start).Milliseconds())
+	return res, err
 }