envoy: refactor envoy embedding (#2296)

* envoy: add full version

* remove unused import

* envoy: refactor envoy embedding

* fix lint

* commit ignored files

* maybe fix test

.github/goreleaser.yaml

@@ -40,9 +40,6 @@ builds:
         - cmd: ./scripts/get-envoy.bash
           env:
             - TARGET={{ .Os }}-{{ .Arch }}
-            - DIR={{ dir .Path }}
-      post:
-        - cmd: ./scripts/embed-envoy.bash {{ .Path }}
 
   - id: pomerium-cli
     main: ./cmd/pomerium-cli

Makefile

@@ -45,7 +45,7 @@ generate-mocks: ## Generate mocks
 	@echo "==> $@"
 	@go run github.com/golang/mock/mockgen -destination internal/directory/auth0/mock_auth0/mock.go github.com/pomerium/pomerium/internal/directory/auth0 RoleManager
 
-.PHONY: build-lint
+.PHONY: deps-lint
 deps-lint: ## Install lint dependencies
 	@echo "==> $@"
 	./scripts/get-envoy.bash
@@ -85,18 +85,16 @@ build: ## Builds dynamic executables and/or packages.
 	@echo "==> $@"
 	./scripts/get-envoy.bash
 	@CGO_ENABLED=0 GO111MODULE=on $(GO) build -tags "$(BUILDTAGS)" ${GO_LDFLAGS} -o $(BINDIR)/$(NAME) ./cmd/"$(NAME)"
-	./scripts/embed-envoy.bash $(BINDIR)/$(NAME)
 
 .PHONY: build-debug
 build-debug: ## Builds binaries appropriate for debugging
 	@echo "==> $@"
 	./scripts/get-envoy.bash
-	@CGO_ENABLED=0 GO111MODULE=on $(GO) build -gcflags="all=-N -l" -ldflags="-X github.com/pomerium/pomerium/internal/envoy.Checksum=$$(cat ./bin/envoy.sha256 | tr -d '\n')" -o $(BINDIR)/$(NAME) ./cmd/"$(NAME)"
+	@CGO_ENABLED=0 GO111MODULE=on $(GO) build -gcflags="all=-N -l" -o $(BINDIR)/$(NAME) ./cmd/"$(NAME)"
-	./scripts/embed-envoy.bash $(BINDIR)/$(NAME)
 
 
 .PHONY: lint
-lint: ## Verifies `golint` passes.
+lint: deps-lint ## Verifies `golint` passes.
 	@echo "==> $@"
 	@golangci-lint run ./...
 

config/envoyconfig/filemgr/filemgr.go

@@ -56,6 +56,10 @@ func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_
 
 // ClearCache clears the file cache.
 func (mgr *Manager) ClearCache() {
+	if _, err := os.Stat(mgr.cfg.cacheDir); os.IsNotExist(err) {
+		return
+	}
+
 	err := filepath.Walk(mgr.cfg.cacheDir, func(p string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return err

go.mod

@@ -32,7 +32,6 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7
 	github.com/martinlindhe/base36 v1.1.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2
@@ -75,7 +74,6 @@ require (
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/auth0.v5 v5.18.0
-	gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -489,8 +489,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
-github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -1396,8 +1394,6 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d h1:YjTGSRV59gG1DHCq68v2B771I9dGFxvMkugf7OKglpk=
-gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d/go.mod h1:kbUs813+JgwKQdecaTv87br/FZUaSEuPj8vbr2vq8sY=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=

internal/cmd/pomerium/pomerium_test.go

@@ -10,7 +10,6 @@ import (
 
 func Test_run(t *testing.T) {
 	os.Clearenv()
-	t.Parallel()
 	tests := []struct {
 		name           string
 		configFileFlag string

internal/envoy/embed.go

@@ -1,15 +1,17 @@
 package envoy
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/natefinch/atomic"
-	resources "gopkg.in/cookieo9/resources-go.v2"
 
+	"github.com/pomerium/pomerium/internal/envoy/files"
 	"github.com/pomerium/pomerium/internal/log"
 )
 
@@ -18,51 +20,49 @@ const (
 	embeddedDirectoryPermissions fs.FileMode = 0o755
 )
 
-var embeddedFilesBaseDirectory = filepath.Join(os.TempDir(), "pomerium-embedded-files")
+var (
+	embeddedFilesBaseDirectory = filepath.Join(os.TempDir(), "pomerium-embedded-files")
+	extractEmbeddedEnvoyOnce   sync.Once
+)
 
 func extractEmbeddedEnvoy(ctx context.Context) (outPath string, err error) {
-	exePath, err := resources.ExecutablePath()
+	extractEmbeddedEnvoyOnce.Do(func() {
-	if err != nil {
+		// clean up our base directory before starting
-		return "", fmt.Errorf("error finding executable path: %w", err)
+		err = os.RemoveAll(embeddedFilesBaseDirectory)
-	}
+		if err != nil {
-	bundle, err := resources.OpenZip(exePath)
+			err = fmt.Errorf("error cleaning embedded file directory: (directory=%s): %w", embeddedFilesBaseDirectory, err)
-	if err != nil {
+			return
-		return "", fmt.Errorf("error opening binary zip file: %w", err)
+		}
-	}
-	defer bundle.Close()
 
-	rc, err := bundle.Open("envoy")
+		// create known directory base to clean at startup
-	if err != nil {
+		err = os.MkdirAll(embeddedFilesBaseDirectory, embeddedDirectoryPermissions)
-		return "", fmt.Errorf("error opening embedded envoy binary: %w", err)
+		if err != nil {
-	}
+			err = fmt.Errorf("error creating embedded file directory: (directory=%s): %w", embeddedFilesBaseDirectory, err)
-	defer rc.Close()
+			return
+		}
 
-	// clean up our base directory before starting
+		// build a random temp directory inside our base directory to guarantee permissions
-	err = os.RemoveAll(embeddedFilesBaseDirectory)
+		var tmpDir string
-	if err != nil {
+		tmpDir, err = os.MkdirTemp(embeddedFilesBaseDirectory, "envoy-")
-		return "", fmt.Errorf("error cleaning embedded file directory: (directory=%s): %w", embeddedFilesBaseDirectory, err)
+		if err != nil {
-	}
+			err = fmt.Errorf("error creating embedded file tmp directory: (directory=%s): %w", embeddedFilesBaseDirectory, err)
+			return
+		}
 
-	// create known directory base to clean at startup
+		outPath = filepath.Join(tmpDir, "envoy")
-	err = os.MkdirAll(embeddedFilesBaseDirectory, embeddedDirectoryPermissions)
-	if err != nil {
-		return "", fmt.Errorf("error creating embedded file directory: (directory=%s): %w", embeddedFilesBaseDirectory, err)
-	}
 
-	// build a random temp directory inside our base directory to guarantee permissions
+		log.Info(ctx).Str("path", outPath).Msg("extracting envoy binary")
-	tmpDir, err := os.MkdirTemp(embeddedFilesBaseDirectory, "envoy-")
+		err = atomic.WriteFile(outPath, bytes.NewReader(files.Binary()))
-	outPath = filepath.Join(tmpDir, "envoy")
+		if err != nil {
+			err = fmt.Errorf("error extracting embedded envoy binary to temporary directory (path=%s): %w", outPath, err)
+			return
+		}
 
-	log.Info(ctx).Str("path", outPath).Msg("extracting envoy binary")
+		err = os.Chmod(outPath, embeddedEnvoyPermissions)
-	err = atomic.WriteFile(outPath, rc)
+		if err != nil {
-	if err != nil {
+			err = fmt.Errorf("error chmoding embedded envoy binary: %w", err)
-		return "", fmt.Errorf("error extracting embedded envoy binary to temporary directory (path=%s): %w", outPath, err)
+			return
-	}
+		}
-
+	})
-	err = os.Chmod(outPath, embeddedEnvoyPermissions)
+	return outPath, err
-	if err != nil {
-		return "", fmt.Errorf("error chmoding embedded envoy binary: %w", err)
-	}
-
-	return outPath, nil
 }

internal/envoy/envoy.go

@@ -121,6 +121,8 @@ func NewServer(ctx context.Context, src config.Source, grpcPort, httpPort string
 
 // Close kills any underlying envoy process.
 func (srv *Server) Close() error {
+	srv.monitorProcessCancel()
+
 	srv.mu.Lock()
 	defer srv.mu.Unlock()
 

internal/envoy/envoy_linux.go

@@ -33,6 +33,7 @@ func (srv *Server) runProcessCollector(ctx context.Context) {
 	if err := view.Register(pc.Views()...); err != nil {
 		log.Error(ctx).Err(err).Msg("failed to register envoy process metric views")
 	}
+	defer view.Unregister(pc.Views()...)
 
 	const collectInterval = time.Second * 10
 	ticker := time.NewTicker(collectInterval)

internal/envoy/files/files.go

@@ -6,11 +6,10 @@ import (
 	"strings"
 )
 
-//go:embed envoy.sha256
+// Binary returns the raw envoy binary bytes.
-var rawChecksum string
+func Binary() []byte {
-
+	return rawBinary
-//go:embed envoy.version
+}
-var rawVersion string
 
 // Checksum returns the checksum for the embedded envoy binary.
 func Checksum() string {

internal/envoy/files/files_darwin.go

@@ -0,0 +1,14 @@
+//+build darwin
+
+package files
+
+import _ "embed" // embed
+
+//go:embed envoy-darwin-amd64
+var rawBinary []byte
+
+//go:embed envoy-darwin-amd64.sha256
+var rawChecksum string
+
+//go:embed envoy-darwin-amd64.version
+var rawVersion string

internal/envoy/files/files_linux_amd64.go

@@ -0,0 +1,14 @@
+//+build linux,amd64
+
+package files
+
+import _ "embed" // embed
+
+//go:embed envoy-linux-amd64
+var rawBinary []byte
+
+//go:embed envoy-linux-amd64.sha256
+var rawChecksum string
+
+//go:embed envoy-linux-amd64.version
+var rawVersion string

internal/envoy/files/files_linux_arm64.go

@@ -0,0 +1,14 @@
+//+build linux,arm64
+
+package files
+
+import _ "embed" // embed
+
+//go:embed envoy-linux-arm64
+var rawBinary []byte
+
+//go:embed envoy-linux-arm64.sha256
+var rawChecksum string
+
+//go:embed envoy-linux-arm64.version
+var rawVersion string

scripts/embed-envoy.bash

@@ -1,17 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-BINARY=$1
-DIR=$(dirname "${BINARY}")
-
-(
-  cd "$DIR"
-  zip envoy.zip envoy
-)
-
-echo "appending $DIR/envoy.zip to ${BINARY}"
-
-if [ "$(unzip -z -qq "$BINARY" 2>&1)" != "" ]; then
-  cat "$DIR/envoy.zip" >>"${BINARY}"
-fi
-zip -A "${BINARY}"

scripts/get-envoy.bash

@@ -6,7 +6,7 @@ export PATH
 
 _project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
 _envoy_version=1.17.3
-_dir="${DIR:-"$_project_root/bin"}"
+_dir="$_project_root/internal/envoy/files"
 _target="${TARGET:-"$(go env GOOS)-$(go env GOARCH)"}"
 
 # until m1 macs are supported, fallback to x86 and use rosetta
@@ -16,31 +16,20 @@ fi
 
 _url="https://github.com/pomerium/envoy-binaries/releases/download/v${_envoy_version}/envoy-${_target}"
 
-# create the directory if it doesn't exist
-mkdir -p "$_dir"
-
-# download the shasum of the binary
 curl \
   --compressed \
   --silent \
   --location \
+  --time-cond "$_dir/envoy-$_target" \
+  --output "$_dir/envoy-$_target" \
+  "$_url"
+
+curl \
+  --compressed \
+  --silent \
+  --location \
+  --time-cond "$_dir/envoy-$_target.sha256" \
   --output "$_dir/envoy-$_target.sha256" \
   "$_url.sha256"
 
-# if the shasum doesn't match (or the binary doesn't exist), re-download
+echo "$_envoy_version" > "$_dir/envoy-$_target.version"
-if ! (cd "$_dir" && shasum -c "envoy-$_target.sha256" >/dev/null 2>&1) ; then
-  curl \
-    --compressed \
-    --silent \
-    --location \
-    --output "$_dir/envoy-$_target" \
-    "$_url"
-fi
-
-# save the bare name
-cp -f "$_dir/envoy-$_target" "$_dir/envoy"
-cp -f "$_dir/envoy-$_target.sha256" "$_dir/envoy.sha256"
-
-# save to the embedded files in the envoy package
-cp -f "$_dir/envoy-$_target.sha256" "$_project_root/internal/envoy/files/envoy.sha256"
-echo "$_envoy_version" > "$_project_root/internal/envoy/files/envoy.version"