core: more metrics (#5629)

## Summary
Add some more metrics:

- Authenticate token verification
- Authorization log duration
- Authorization evaluator and header evaluator
- IDP token session creator

HTTP and gRPC endpoints are already instrumented via middleware, which
covers authenticate, proxy and databroker endpoints. Postgres is also
already instrumented using `otelpgx`.

## Related issues
-
[ENG-2407](https://linear.app/pomerium/issue/ENG-2407/add-additional-metrics-and-tracing-spans-to-pomerium)


## Checklist

- [x] reference any related issues
- [ ] updated unit tests
- [ ] add appropriate label (`enhancement`, `bug`, `breaking`,
`dependencies`, `ci`)
- [x] ready for review

authenticate/handlers_verify.go

@@ -3,6 +3,7 @@ package authenticate
 import (
 	"encoding/json"
 	"net/http"
+	"time"
 
 	"github.com/pomerium/pomerium/internal/httputil"
 	"github.com/pomerium/pomerium/internal/log"
@@ -10,6 +11,10 @@ import (
 )
 
 func (a *Authenticate) verifyAccessToken(w http.ResponseWriter, r *http.Request) error {
+	start := time.Now()
+
+	a.accessTokenVerificationCount.Add(r.Context(), 1)
+
 	var req authenticateapi.VerifyAccessTokenRequest
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
@@ -24,9 +29,11 @@ func (a *Authenticate) verifyAccessToken(w http.ResponseWriter, r *http.Request)
 	var res authenticateapi.VerifyTokenResponse
 	claims, err := authenticator.VerifyAccessToken(r.Context(), req.AccessToken)
 	if err == nil {
+		a.accessTokenValidVerificationCount.Add(r.Context(), 1)
 		res.Valid = true
 		res.Claims = claims
 	} else {
+		a.accessTokenInvalidVerificationCount.Add(r.Context(), 1)
 		res.Valid = false
 		log.Ctx(r.Context()).Info().
 			Err(err).
@@ -39,10 +46,16 @@ func (a *Authenticate) verifyAccessToken(w http.ResponseWriter, r *http.Request)
 		return err
 	}
 
+	a.accessTokenVerificationDuration.Record(r.Context(), time.Since(start).Milliseconds())
+
 	return nil
 }
 
 func (a *Authenticate) verifyIdentityToken(w http.ResponseWriter, r *http.Request) error {
+	start := time.Now()
+
+	a.identityTokenVerificationCount.Add(r.Context(), 1)
+
 	var req authenticateapi.VerifyIdentityTokenRequest
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
@@ -57,9 +70,11 @@ func (a *Authenticate) verifyIdentityToken(w http.ResponseWriter, r *http.Reques
 	var res authenticateapi.VerifyTokenResponse
 	claims, err := authenticator.VerifyIdentityToken(r.Context(), req.IdentityToken)
 	if err == nil {
+		a.identityTokenValidVerificationCount.Add(r.Context(), 1)
 		res.Valid = true
 		res.Claims = claims
 	} else {
+		a.identityTokenInvalidVerificationCount.Add(r.Context(), 1)
 		res.Valid = false
 		log.Ctx(r.Context()).Info().
 			Err(err).
@@ -72,5 +87,7 @@ func (a *Authenticate) verifyIdentityToken(w http.ResponseWriter, r *http.Reques
 		return err
 	}
 
+	a.identityTokenVerificationDuration.Record(r.Context(), time.Since(start).Milliseconds())
+
 	return nil
 }