config: allow reloading of telemetry settings (#1255)

* metrics: support dynamic configuration settings

* add test

* trace: update configuration when settings change

* config: allow logging options to be configured when settings change

* envoy: allow changing log settings

* fix unexpected doc change

* fix tests

* pick a port at random

* update based on review

internal/envoy/envoy.go

@@ -32,6 +32,7 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/internal/log"
 	"github.com/pomerium/pomerium/internal/telemetry"
+	"github.com/pomerium/pomerium/internal/telemetry/trace"
 )
 
 const (
@@ -46,7 +47,6 @@ type Server struct {
 
 	grpcPort, httpPort string
 	opts               *config.Options
-	logLevel           string
 }
 
 // NewServer creates a new server with traffic routed by envoy.
@@ -64,12 +64,6 @@ func NewServer(opts *config.Options, grpcPort, httpPort string) (*Server, error)
 		opts:     opts,
 	}
 
-	if srv.opts.ProxyLogLevel != "" {
-		srv.logLevel = srv.opts.ProxyLogLevel
-	} else {
-		srv.logLevel = srv.opts.LogLevel
-	}
-
 	err = srv.writeConfig()
 	if err != nil {
 		return nil, fmt.Errorf("error writing initial envoy configuration: %w", err)
@@ -88,7 +82,7 @@ func (srv *Server) Run(ctx context.Context) error {
 
 	srv.cmd = exec.CommandContext(ctx, envoyPath,
 		"-c", configFileName,
-		"--log-level", srv.logLevel,
+		"--log-level", "trace",
 		"--log-format", "[LOG_FORMAT]%l--%n--%v",
 		"--log-format-escaped",
 		"--disable-hot-restart",
@@ -268,7 +262,7 @@ func (srv *Server) addTraceConfig(traceOpts *config.TracingOptions, bootCfg *env
 	}
 
 	// We only support zipkin in envoy currently
-	if traceOpts.Provider != config.ZipkinTracingProviderName {
+	if traceOpts.Provider != trace.ZipkinTracingProviderName {
 		return nil
 	}
 
@@ -354,6 +348,11 @@ func (srv *Server) handleLogs(rc io.ReadCloser) {
 			msg = s
 		}
 
+		// ignore empty messages
+		if msg == "" {
+			continue
+		}
+
 		log.WithLevel(lvl).
 			Str("service", "envoy").
 			Str("name", name).