assets: use embed instead of statik (#1960)

* assets: use embed instead of statik

* remove empty line

* maybe fix precommit

.gitattributes

@@ -1 +0,0 @@
-statik.go binary

.github/workflows/test.yaml

@@ -157,6 +157,9 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
       - uses: actions/setup-python@v2
       - uses: pre-commit/action@release
         with:

Makefile

@@ -119,7 +119,6 @@ cover: ## Runs go test with coverage
 	@echo "==> $@"
 	$(GO) test -race -coverprofile=coverage.txt -tags "$(BUILDTAGS)" $(shell $(GO) list ./... | grep -v vendor | grep -v github.com/pomerium/pomerium/integration)
 	@sed -i.bak '/\.pb\.go\:/d' coverage.txt
-	@sed -i.bak '/\/statik\.go\:/d' coverage.txt
 	@sed -i.bak '/\/mock\.go\:/d' coverage.txt
 	@sort -o coverage.txt coverage.txt
 

authorize/evaluator/evaluator.go

@@ -38,7 +38,7 @@ func New(options *config.Options, store *Store) (*Evaluator, error) {
 		return nil, fmt.Errorf("authorize: couldn't create signer: %w", err)
 	}
 
-	authzPolicy, err := readPolicy("/authz.rego")
+	authzPolicy, err := readPolicy()
 	if err != nil {
 		return nil, fmt.Errorf("error loading rego policy: %w", err)
 	}

authorize/evaluator/functions.go

@@ -4,12 +4,10 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/rakyll/statik/fs"
 
-	_ "github.com/pomerium/pomerium/authorize/evaluator/opa/policy" // load static assets
+	"github.com/pomerium/pomerium/authorize/evaluator/opa"
 	"github.com/pomerium/pomerium/internal/log"
 )
 
@@ -67,17 +65,6 @@ func parseCertificate(pemStr string) (*x509.Certificate, error) {
 	return x509.ParseCertificate(block.Bytes)
 }
 
-const statikNamespace = "rego"
+func readPolicy() ([]byte, error) {
-
+	return opa.FS.ReadFile("policy/authz.rego")
-func readPolicy(fn string) ([]byte, error) {
-	statikFS, err := fs.NewWithNamespace(statikNamespace)
-	if err != nil {
-		return nil, err
-	}
-	r, err := statikFS.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return ioutil.ReadAll(r)
 }

authorize/evaluator/opa/opa.go

@@ -2,5 +2,8 @@
 // decisions.
 package opa
 
-//go:generate go run github.com/rakyll/statik -m -src=./policy -include=*.rego -ns rego -p policy
+import "embed"
-//go:generate go fmt ./policy/statik.go
+
+// FS is the filesystem for OPA files.
+//go:embed policy
+var FS embed.FS

authorize/evaluator/opa_test.go

@@ -35,7 +35,7 @@ func TestOPA(t *testing.T) {
 	require.NoError(t, err)
 
 	eval := func(policies []config.Policy, data []proto.Message, req *Request, isValidClientCertificate bool) rego.Result {
-		authzPolicy, err := readPolicy("/authz.rego")
+		authzPolicy, err := readPolicy()
 		require.NoError(t, err)
 		store := NewStoreFromProtos(data...)
 		store.UpdateIssuer("authenticate.example.com")

go.mod

@@ -1,6 +1,6 @@
 module github.com/pomerium/pomerium
 
-go 1.14
+go 1.16
 
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.1
@@ -45,7 +45,6 @@ require (
 	github.com/pomerium/csrf v1.7.0
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/procfs v0.2.0
-	github.com/rakyll/statik v0.1.7
 	github.com/rjeczalik/notify v0.9.3-0.20201210012515-e2a77dcc14cf
 	github.com/rs/cors v1.7.0
 	github.com/rs/zerolog v1.20.0

go.sum

@@ -159,7 +159,6 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -529,8 +528,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/statsd_exporter v0.15.0 h1:UiwC1L5HkxEPeapXdm2Ye0u1vUJfTj7uwT5yydYpa1E=
 github.com/prometheus/statsd_exporter v0.15.0/go.mod h1:Dv8HnkoLQkeEjkIE4/2ndAA7WL1zHKK7WMqFQqu72rw=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
-github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=

internal/controlplane/xds_lua.go

@@ -1,16 +1,12 @@
 package controlplane
 
 import (
-	"os"
+	"embed"
-
+	"io/fs"
-	"github.com/rakyll/statik/fs"
-
-	// include luascripts source code
-	_ "github.com/pomerium/pomerium/internal/controlplane/luascripts"
 )
 
-//go:generate go run github.com/rakyll/statik -m -src=./luascripts -include=*.lua -p luascripts -ns luascripts
+//go:embed luascripts
-//go:generate go fmt ./luascripts/statik.go
+var luaFS embed.FS
 
 var luascripts struct {
 	ExtAuthzSetCookie        string
@@ -20,28 +16,23 @@ var luascripts struct {
 }
 
 func init() {
-	hfs, err := fs.NewWithNamespace("luascripts")
-	if err != nil {
-		panic(err)
-	}
-
 	fileToField := map[string]*string{
-		"/clean-upstream.lua":             &luascripts.CleanUpstream,
+		"luascripts/clean-upstream.lua":             &luascripts.CleanUpstream,
-		"/ext-authz-set-cookie.lua":       &luascripts.ExtAuthzSetCookie,
+		"luascripts/ext-authz-set-cookie.lua":       &luascripts.ExtAuthzSetCookie,
-		"/remove-impersonate-headers.lua": &luascripts.RemoveImpersonateHeaders,
+		"luascripts/remove-impersonate-headers.lua": &luascripts.RemoveImpersonateHeaders,
-		"/fix-misdirected.lua":            &luascripts.FixMisdirected,
+		"luascripts/fix-misdirected.lua":            &luascripts.FixMisdirected,
 	}
 
-	err = fs.Walk(hfs, "/", func(p string, fi os.FileInfo, err error) error {
+	err := fs.WalkDir(luaFS, "luascripts", func(p string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if fi.IsDir() {
+		if d.IsDir() {
 			return nil
 		}
 
-		bs, err := fs.ReadFile(hfs, p)
+		bs, err := luaFS.ReadFile(p)
 		if err != nil {
 			return err
 		}

internal/frontend/templates.go

@@ -2,56 +2,49 @@
 // html templates.
 package frontend
 
-//go:generate go run github.com/rakyll/statik -m -src=./assets -include=*.svg,*.html,*.css,*.js -ns web
-//go:generate go fmt statik/statik.go
-
 import (
+	"embed"
 	"encoding/base64"
 	"fmt"
 	"html/template"
-	"io/ioutil"
+	"io/fs"
 	"mime"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 	"time"
-
-	"github.com/rakyll/statik/fs"
-
-	_ "github.com/pomerium/pomerium/internal/frontend/statik" // load static assets
 )
 
-const statikNamespace = "web"
+// FS is the frontend assets file system.
+//go:embed assets
+var FS embed.FS
 
 // NewTemplates loads pomerium's templates. Panics on failure.
 func NewTemplates() (*template.Template, error) {
-	statikFS, err := fs.NewWithNamespace(statikNamespace)
+	assetsFS, err := fs.Sub(FS, "assets")
 	if err != nil {
-		return nil, fmt.Errorf("internal/frontend: error creating new file system: %w", err)
+		return nil, err
 	}
 
 	dataURLs := map[string]template.URL{}
+	err = fs.WalkDir(assetsFS, ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
 
-	err = fs.Walk(statikFS, "/", func(filePath string, fileInfo os.FileInfo, _ error) error {
+		if d.IsDir() {
-		if fileInfo.IsDir() {
 			return nil
 		}
 
-		file, err := statikFS.Open(filePath)
+		bs, err := fs.ReadFile(assetsFS, p)
 		if err != nil {
-			return fmt.Errorf("internal/frontend: error opening %s: %w", filePath, err)
+			return fmt.Errorf("internal/frontend: error reading %s: %w", p, err)
-		}
-		defer file.Close()
-
-		bs, err := ioutil.ReadAll(file)
-		if err != nil {
-			return fmt.Errorf("internal/frontend: error reading %s: %w", filePath, err)
 		}
 
 		encoded := base64.StdEncoding.EncodeToString(bs)
-		dataURLs[filePath] = template.URL(fmt.Sprintf(
+		dataURLs[p] = template.URL(fmt.Sprintf(
-			"data:%s;base64,%s", mime.TypeByExtension(path.Ext(filePath)), encoded))
+			"data:%s;base64,%s", mime.TypeByExtension(path.Ext(p)), encoded))
 
 		return nil
 	})
@@ -70,27 +63,27 @@ func NewTemplates() (*template.Template, error) {
 			return template.HTMLAttr(fmt.Sprint(arg))
 		},
 		"dataURL": func(p string) template.URL {
-			return dataURLs[strings.TrimPrefix(p, "/.pomerium/assets")]
+			return dataURLs[strings.TrimPrefix(p, "/.pomerium/assets/")]
 		},
 		"formatTime": func(tm time.Time) string {
 			return tm.Format("2006-01-02 15:04:05 MST")
 		},
 	})
 
-	err = fs.Walk(statikFS, "/html", func(filePath string, fileInfo os.FileInfo, err error) error {
+	err = fs.WalkDir(assetsFS, "html", func(p string, d os.DirEntry, err error) error {
-		if !fileInfo.IsDir() {
+		if err != nil {
-			file, err := statikFS.Open(filePath)
+			return err
+		}
+
+		if !d.IsDir() {
+			bs, err := fs.ReadFile(assetsFS, p)
 			if err != nil {
-				return fmt.Errorf("internal/frontend: error opening %s: %w", filePath, err)
+				return fmt.Errorf("internal/frontend: error reading %s: %w", p, err)
 			}
 
-			buf, err := ioutil.ReadAll(file)
+			_, err = t.Parse(string(bs))
 			if err != nil {
-				return fmt.Errorf("internal/frontend: error reading %s: %w", filePath, err)
+				return fmt.Errorf("internal/frontend: error parsing template %s: %w", p, err)
-			}
-			_, err = t.Parse(string(buf))
-			if err != nil {
-				return fmt.Errorf("internal/frontend: error parsing template %s: %w", filePath, err)
 			}
 		}
 		return nil
@@ -105,9 +98,9 @@ func NewTemplates() (*template.Template, error) {
 // MustAssetHandler wraps a call to the embedded static file system and panics
 // if the error is non-nil. It is intended for use in variable initializations
 func MustAssetHandler() http.Handler {
-	statikFS, err := fs.NewWithNamespace(statikNamespace)
+	assetsFS, err := fs.Sub(FS, "assets")
 	if err != nil {
 		panic(err)
 	}
-	return http.FileServer(statikFS)
+	return http.FileServer(http.FS(assetsFS))
 }