zero/connect: add re-run health checks command (#5219)

* zero/connect: add run health checks and shutdown commands

* fix proto

* trigger re-run on command

* add handler

* rename runPeriodicHealthChecksLeased
This commit is contained in:
Denis Mishin 2024-08-22 16:17:53 -04:00 committed by GitHub
parent 6e766233c7
commit 0503b41108
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 220 additions and 104 deletions

View file

@ -16,6 +16,7 @@ import (
sdk "github.com/pomerium/pomerium/internal/zero/api"
"github.com/pomerium/pomerium/internal/zero/bootstrap"
"github.com/pomerium/pomerium/internal/zero/bootstrap/writers"
connect_mux "github.com/pomerium/pomerium/internal/zero/connect-mux"
"github.com/pomerium/pomerium/internal/zero/healthcheck"
"github.com/pomerium/pomerium/internal/zero/reconciler"
"github.com/pomerium/pomerium/internal/zero/telemetry"
@ -157,7 +158,7 @@ func (c *controller) runZeroControlLoop(ctx context.Context) error {
WithLease(
c.runReconcilerLeased,
c.runSessionAnalyticsLeased,
c.runPeriodicHealthChecksLeased,
c.runHealthChecksLeased,
leaseStatus.MonitorLease,
),
)
@ -193,9 +194,17 @@ func (c *controller) runSessionAnalyticsLeased(ctx context.Context, client datab
})
}
func (c *controller) runPeriodicHealthChecksLeased(ctx context.Context, client databroker.DataBrokerServiceClient) error {
func (c *controller) runHealthChecksLeased(ctx context.Context, client databroker.DataBrokerServiceClient) error {
return retry.WithBackoff(ctx, "zero-healthcheck", func(ctx context.Context) error {
return healthcheck.RunChecks(ctx, c.bootstrapConfig, client)
checker := healthcheck.NewChecker(c.bootstrapConfig, client)
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error { return checker.Run(ctx) })
eg.Go(func() error {
return c.api.Watch(ctx, connect_mux.WithOnRunHealthChecks(func(_ context.Context) {
checker.ForceCheck()
}))
})
return eg.Wait()
})
}

View file

@ -4,9 +4,11 @@ import (
"context"
"github.com/rs/zerolog"
"google.golang.org/protobuf/encoding/protojson"
"github.com/pomerium/pomerium/internal/log"
connect_mux "github.com/pomerium/pomerium/internal/zero/connect-mux"
"github.com/pomerium/pomerium/pkg/zero/connect"
)
func (c *controller) RunConnectLog(ctx context.Context) error {
@ -25,5 +27,12 @@ func (c *controller) RunConnectLog(ctx context.Context) error {
connect_mux.WithOnBundleUpdated(func(_ context.Context, key string) {
logger.Debug().Str("key", key).Msg("bundle updated")
}),
connect_mux.WithOnRunHealthChecks(func(_ context.Context) {
logger.Debug().Msg("run health checks")
}),
connect_mux.WithOnTelemetryRequested(func(_ context.Context, req *connect.TelemetryRequest) {
data, _ := protojson.Marshal(req)
logger.Debug().RawJSON("request", data).Msg("telemetry requested")
}),
)
}