envoy: disable hot-reload for macos (#2259)

* envoy: disable hotload for macos

* lint

* envoy: ignore error when retrieving process status

internal/envoy/envoy.go

@@ -15,7 +15,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -59,7 +58,6 @@ type Server struct {
 	builder            *envoyconfig.Builder
 	grpcPort, httpPort string
 	envoyPath          string
-	restartEpoch       int
 
 	monitorProcessCancel context.CancelFunc
 
@@ -190,20 +188,8 @@ func (srv *Server) run(ctx context.Context, cfg *config.Config) error {
 		"--log-format-escaped",
 	}
 
-	if baseID, ok := readBaseID(); ok {
+	exePath, args := srv.prepareRunEnvoyCommand(ctx, args)
-		args = append(args, "--base-id", strconv.Itoa(baseID), "--restart-epoch", strconv.Itoa(srv.restartEpoch))
+	cmd := exec.Command(exePath, args...)
-	} else {
-		args = append(args, "--use-dynamic-base-id", "--base-id-path", baseIDPath)
-	}
-	srv.restartEpoch++
-
-	var cmd *exec.Cmd
-	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
-		// until m1 macs are supported by envoy, fallback to x86 and use rosetta
-		cmd = exec.Command("arch", append([]string{"-x86_64", srv.envoyPath}, args...)...) // #nosec
-	} else {
-		cmd = exec.Command(srv.envoyPath, args...) // #nosec
-	}
 	cmd.Dir = srv.wd
 
 	stderr, err := cmd.StderrPipe()
@@ -233,14 +219,6 @@ func (srv *Server) run(ctx context.Context, cfg *config.Config) error {
 	monitorProcessCtx, srv.monitorProcessCancel = context.WithCancel(context.Background())
 	go srv.monitorProcess(monitorProcessCtx, int32(cmd.Process.Pid))
 
-	// release the previous process so we can hot-reload
-	if srv.cmd != nil && srv.cmd.Process != nil {
-		log.Info(ctx).Msg("envoy: releasing envoy process for hot-reload")
-		err := srv.cmd.Process.Release()
-		if err != nil {
-			log.Warn(ctx).Err(err).Str("service", "envoy").Msg("envoy: failed to release envoy process for hot-reload")
-		}
-	}
 	srv.cmd = cmd
 
 	return nil
@@ -406,7 +384,7 @@ func (srv *Server) monitorProcess(ctx context.Context, pid int32) {
 
 		running, err := proc.IsRunningWithContext(ctx)
 		if err != nil {
-			log.Fatal().Err(err).
+			log.Error(ctx).Err(err).
 				Int32("pid", pid).
 				Msg("envoy: error retrieving subprocess status")
 		} else if !running {

internal/envoy/envoy_darwin.go

@@ -4,7 +4,10 @@ package envoy
 
 import (
 	"context"
+	"runtime"
 	"syscall"
+
+	"github.com/pomerium/pomerium/internal/log"
 )
 
 var sysProcAttr = &syscall.SysProcAttr{
@@ -12,3 +15,20 @@ var sysProcAttr = &syscall.SysProcAttr{
 }
 
 func (srv *Server) runProcessCollector(ctx context.Context) {}
+
+func (srv *Server) prepareRunEnvoyCommand(ctx context.Context, sharedArgs []string) (exePath string, args []string) {
+	if srv.cmd != nil && srv.cmd.Process != nil {
+		log.Info(ctx).Msg("envoy: terminating previous envoy process")
+		_ = srv.cmd.Process.Kill()
+	}
+
+	args = make([]string, len(sharedArgs))
+	copy(args, sharedArgs)
+
+	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+		// until m1 macs are supported by envoy, fallback to x86 and use rosetta
+		return "arch", append([]string{"-x86_64", srv.envoyPath}, args...)
+	}
+
+	return srv.envoyPath, args
+}

internal/envoy/envoy_linux.go

@@ -4,6 +4,9 @@ package envoy
 
 import (
 	"context"
+	"io/ioutil"
+	"strconv"
+	"sync"
 	"syscall"
 	"time"
 
@@ -13,6 +16,13 @@ import (
 	"github.com/pomerium/pomerium/internal/telemetry/metrics"
 )
 
+const baseIDPath = "/tmp/pomerium-envoy-base-id"
+
+var restartEpoch struct {
+	sync.Mutex
+	value int
+}
+
 var sysProcAttr = &syscall.SysProcAttr{
 	Setpgid:   true,
 	Pdeathsig: syscall.SIGTERM,
@@ -50,3 +60,43 @@ func (srv *Server) runProcessCollector(ctx context.Context) {
 		}
 	}
 }
+
+func (srv *Server) prepareRunEnvoyCommand(ctx context.Context, sharedArgs []string) (exePath string, args []string) {
+	// release the previous process so we can hot-reload
+	if srv.cmd != nil && srv.cmd.Process != nil {
+		log.Info(ctx).Msg("envoy: releasing envoy process for hot-reload")
+		err := srv.cmd.Process.Release()
+		if err != nil {
+			log.Warn(ctx).Err(err).Str("service", "envoy").Msg("envoy: failed to release envoy process for hot-reload")
+		}
+	}
+
+	args = make([]string, len(sharedArgs))
+	copy(args, sharedArgs)
+
+	restartEpoch.Lock()
+	if baseID, ok := readBaseID(); ok {
+		args = append(args, "--base-id", strconv.Itoa(baseID), "--restart-epoch", strconv.Itoa(restartEpoch.value))
+		restartEpoch.value++
+	} else {
+		args = append(args, "--use-dynamic-base-id", "--base-id-path", baseIDPath)
+		restartEpoch.value = 1
+	}
+	restartEpoch.Unlock()
+
+	return srv.envoyPath, args
+}
+
+func readBaseID() (int, bool) {
+	bs, err := ioutil.ReadFile(baseIDPath)
+	if err != nil {
+		return 0, false
+	}
+
+	baseID, err := strconv.Atoi(string(bs))
+	if err != nil {
+		return 0, false
+	}
+
+	return baseID, true
+}

internal/envoy/envoy_other.go

@@ -4,6 +4,20 @@ package envoy
 
 import (
 	"context"
+
+	"github.com/pomerium/pomerium/internal/log"
 )
 
 func (srv *Server) runProcessCollector(ctx context.Context) {}
+
+func (srv *Server) prepareRunEnvoyCommand(ctx context.Context, sharedArgs []string) (exePath string, args []string) {
+	if srv.cmd != nil && srv.cmd.Process != nil {
+		log.Info(ctx).Msg("envoy: terminating previous envoy process")
+		_ = srv.cmd.Process.Kill()
+	}
+
+	args = make([]string, len(sharedArgs))
+	copy(args, sharedArgs)
+
+	return srv.envoyPath, args
+}

internal/envoy/misc.go

@@ -2,15 +2,12 @@ package envoy
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net"
 	"strconv"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 )
 
-const baseIDPath = "/tmp/pomerium-envoy-base-id"
-
 func firstNonEmpty(args ...string) string {
 	for _, a := range args {
 		if a != "" {
@@ -20,20 +17,6 @@ func firstNonEmpty(args ...string) string {
 	return ""
 }
 
-func readBaseID() (int, bool) {
-	bs, err := ioutil.ReadFile(baseIDPath)
-	if err != nil {
-		return 0, false
-	}
-
-	baseID, err := strconv.Atoi(string(bs))
-	if err != nil {
-		return 0, false
-	}
-
-	return baseID, true
-}
-
 // ParseAddress parses a string address into an envoy address.
 func ParseAddress(raw string) (*envoy_config_core_v3.Address, error) {
 	if host, portstr, err := net.SplitHostPort(raw); err == nil {