diff --git a/internal/autocert/manager.go b/internal/autocert/manager.go
index 8f54332ca..6a5c0796f 100644
--- a/internal/autocert/manager.go
+++ b/internal/autocert/manager.go
@@ -52,9 +52,10 @@ type Manager struct {
 	acmeMgr   atomic.Pointer[certmagic.ACMEIssuer]
 	srv       *http.Server
 
-	acmeTLSALPNLock   sync.Mutex
-	acmeTLSALPNPort   string
-	acmeTLSALPNConfig *tls.Config
+	acmeTLSALPNLock     sync.Mutex
+	acmeTLSALPNPort     string
+	acmeTLSALPNListener net.Listener
+	acmeTLSALPNConfig   *tls.Config
 
 	*ocspCache
 
@@ -158,7 +159,7 @@ func (mgr *Manager) getCertMagicConfig(ctx context.Context, cfg *config.Config)
 		}
 	}
 	acmeMgr := certmagic.NewACMEIssuer(mgr.certmagic, mgr.acmeTemplate)
-	acmeMgr.DisableHTTPChallenge = cfg.Options.HTTPRedirectAddr == ""
+	acmeMgr.DisableHTTPChallenge = !shouldEnableHTTPChallenge(cfg)
 	err = configureCertificateAuthority(acmeMgr, cfg.Options.AutocertOptions)
 	if err != nil {
 		return nil, err
@@ -359,6 +360,11 @@ func (mgr *Manager) updateACMETLSALPNServer(ctx context.Context, cfg *config.Con
 	// store the updated port
 	mgr.acmeTLSALPNPort = cfg.ACMETLSALPNPort
 
+	if mgr.acmeTLSALPNListener != nil {
+		_ = mgr.acmeTLSALPNListener.Close()
+		mgr.acmeTLSALPNListener = nil
+	}
+
 	// start the listener
 	addr := net.JoinHostPort("127.0.0.1", cfg.ACMETLSALPNPort)
 	ln, err := net.Listen("tcp", addr)
@@ -366,6 +372,7 @@ func (mgr *Manager) updateACMETLSALPNServer(ctx context.Context, cfg *config.Con
 		log.Error(ctx).Err(err).Msg("failed to run acme tls alpn server")
 		return
 	}
+	mgr.acmeTLSALPNListener = ln
 
 	// accept connections
 	go func() {
@@ -495,3 +502,16 @@ func sourceHostnames(cfg *config.Config) []string {
 
 	return h
 }
+
+func shouldEnableHTTPChallenge(cfg *config.Config) bool {
+	if cfg == nil || cfg.Options == nil {
+		return false
+	}
+
+	_, p, err := net.SplitHostPort(cfg.Options.HTTPRedirectAddr)
+	if err != nil {
+		return false
+	}
+
+	return p == "80"
+}
diff --git a/internal/autocert/manager_test.go b/internal/autocert/manager_test.go
index 20dac7d30..c123d0bf6 100644
--- a/internal/autocert/manager_test.go
+++ b/internal/autocert/manager_test.go
@@ -630,3 +630,23 @@ func Test_configureTrustedRoots(t *testing.T) {
 		})
 	}
 }
+
+func TestShouldEnableHTTPChallenge(t *testing.T) {
+	t.Parallel()
+
+	assert.False(t, shouldEnableHTTPChallenge(nil))
+	assert.False(t, shouldEnableHTTPChallenge(&config.Config{}))
+	assert.False(t, shouldEnableHTTPChallenge(&config.Config{Options: &config.Options{}}))
+	assert.False(t, shouldEnableHTTPChallenge(&config.Config{Options: &config.Options{
+		HTTPRedirectAddr: ":8080",
+	}}))
+	assert.False(t, shouldEnableHTTPChallenge(&config.Config{Options: &config.Options{
+		HTTPRedirectAddr: "127.0.0.1:8080",
+	}}))
+	assert.True(t, shouldEnableHTTPChallenge(&config.Config{Options: &config.Options{
+		HTTPRedirectAddr: ":80",
+	}}))
+	assert.True(t, shouldEnableHTTPChallenge(&config.Config{Options: &config.Options{
+		HTTPRedirectAddr: "127.0.0.1:80",
+	}}))
+}