zero/telemetry: internal envoy stats scraper and metrics producer (#5136)

internal/retry/backoff.go

@@ -2,16 +2,27 @@ package retry
 
 import (
 	"context"
+	"strings"
+	"time"
 
 	"github.com/cenkalti/backoff/v4"
+
+	"github.com/pomerium/pomerium/internal/log"
 )
 
+type serviceName struct{}
+
 // WithBackoff retries the given function with an exponential backoff,
 // stopping when the context is done or the function returns a terminal error.
-func WithBackoff(ctx context.Context, fn func(context.Context) error) error {
+func WithBackoff(ctx context.Context, name string, fn func(context.Context) error) error {
+	name, ctx = getServiceNameContext(ctx, name)
+
+	log.Debug(ctx).Str("service-name", name).Msg("starting")
+	defer log.Debug(ctx).Str("service-name", name).Msg("stopped")
+
 	b := backoff.NewExponentialBackOff()
 	b.MaxElapsedTime = 0
-	return backoff.Retry(
+	return backoff.RetryNotify(
 		func() error {
 			err := fn(ctx)
 			if IsTerminalError(err) {
@@ -20,5 +31,18 @@ func WithBackoff(ctx context.Context, fn func(context.Context) error) error {
 			return err
 		},
 		backoff.WithContext(b, ctx),
+		func(err error, next time.Duration) {
+			log.Warn(ctx).Err(err).Str("service-name", name).Dur("next", next).Msg("retrying")
+		},
 	)
 }
+
+func getServiceNameContext(ctx context.Context, name string) (string, context.Context) {
+	names, ok := ctx.Value(serviceName{}).([]string)
+	if ok {
+		names = append(names, name)
+	} else {
+		names = []string{name}
+	}
+	return strings.Join(names, "."), context.WithValue(ctx, serviceName{}, names)
+}