Proxy envoy metrics through control plane prometheus endpoint (#709)

* Proxy metrics requests to envoy control plane

internal/envoy/envoy.go

@@ -23,6 +23,8 @@ import (
 const (
 	workingDirectoryName = ".pomerium-envoy"
 	configFileName       = "envoy-config.yaml"
+	// EnvoyAdminURL indicates where the envoy control plane is listening
+	EnvoyAdminURL = "http://localhost:9901"
 )
 
 // A Server is a pomerium proxy implemented via envoy.

internal/telemetry/metrics/providers.go

@@ -2,13 +2,21 @@ package metrics
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 
 	ocprom "contrib.go.opencensus.io/exporter/prometheus"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/stats/view"
+
+	"github.com/pomerium/pomerium/internal/envoy"
+	log "github.com/pomerium/pomerium/internal/log"
+	"github.com/pomerium/pomerium/internal/urlutil"
 )
 
+var envoyURL = envoy.EnvoyAdminURL
+
 // PrometheusHandler creates an exporter that exports stats to Prometheus
 // and returns a handler suitable for exporting metrics.
 func PrometheusHandler() (http.Handler, error) {
@@ -26,7 +34,13 @@ func PrometheusHandler() (http.Handler, error) {
 	}
 	view.RegisterExporter(exporter)
 	mux := http.NewServeMux()
-	mux.Handle("/metrics", exporter)
+
+	envoyMetricsURL, err := urlutil.ParseAndValidateURL(fmt.Sprintf("%s/stats/prometheus", envoyURL))
+	if err != nil {
+		return nil, fmt.Errorf("telemetry/metrics: invalid proxy URL: %w", err)
+	}
+
+	mux.Handle("/metrics", newProxyMetricsHandler(exporter, *envoyMetricsURL))
 	return mux, nil
 }
 
@@ -37,3 +51,33 @@ func registerDefaultViews() error {
 	}
 	return view.Register(views...)
 }
+
+// newProxyMetricsHandler creates a subrequest to the envoy control plane for metrics and
+// combines them with our own
+func newProxyMetricsHandler(promHandler http.Handler, envoyURL url.URL) http.HandlerFunc {
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer promHandler.ServeHTTP(w, r)
+
+		r, err := http.NewRequestWithContext(r.Context(), "GET", envoyURL.String(), nil)
+		if err != nil {
+			log.Error().Err(err).Msg("telemetry/metrics: failed to create request for envoy")
+			return
+		}
+
+		resp, err := http.DefaultClient.Do(r)
+		if err != nil {
+			log.Error().Err(err).Msg("telemetry/metrics: fail to fetch proxy metrics")
+			return
+		}
+		defer resp.Body.Close()
+
+		envoyBody, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Error().Err(err).Msg("telemetry/metric: failed to read proxy metrics")
+			return
+		}
+
+		w.Write(envoyBody)
+	}
+}

internal/telemetry/metrics/providers_test.go

@@ -1,29 +1,72 @@
 package metrics
 
 import (
-	"bytes"
 	"io/ioutil"
+	"net/http"
 	"net/http/httptest"
 	"regexp"
 	"testing"
 )
 
-func Test_PrometheusHandler(t *testing.T) {
+func newEnvoyMetricsHandler() http.HandlerFunc {
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+# TYPE envoy_server_initialization_time_ms histogram
+envoy_server_initialization_time_ms_bucket{le="0.5"} 0
+envoy_server_initialization_time_ms_bucket{le="1"} 0
+envoy_server_initialization_time_ms_bucket{le="5"} 0
+envoy_server_initialization_time_ms_bucket{le="10"} 0
+envoy_server_initialization_time_ms_bucket{le="25"} 0
+envoy_server_initialization_time_ms_bucket{le="50"} 0
+envoy_server_initialization_time_ms_bucket{le="100"} 0
+envoy_server_initialization_time_ms_bucket{le="250"} 0
+envoy_server_initialization_time_ms_bucket{le="500"} 1
+envoy_server_initialization_time_ms_bucket{le="1000"} 1
+`))
+	}
+}
+
+func getMetrics(t *testing.T) []byte {
 	h, err := PrometheusHandler()
 	if err != nil {
 		t.Fatal(err)
 	}
-	req := httptest.NewRequest("GET", "http://test.local/metrics", new(bytes.Buffer))
+	req := httptest.NewRequest("GET", "http://test.local/metrics", nil)
 	rec := httptest.NewRecorder()
 	h.ServeHTTP(rec, req)
+
 	resp := rec.Result()
 	b, _ := ioutil.ReadAll(resp.Body)
 
 	if resp == nil || resp.StatusCode != 200 {
 		t.Errorf("Metrics endpoint failed to respond: %s", b)
 	}
+	return b
+}
 
-	if m, _ := regexp.Match("^# HELP .*", b); !m {
+func Test_PrometheusHandler(t *testing.T) {
+
+	t.Run("no envoy", func(t *testing.T) {
+		b := getMetrics(t)
+
+		if m, _ := regexp.Match(`(?m)^# HELP .*`, b); !m {
 			t.Errorf("Metrics endpoint did not contain any help messages: %s", b)
 		}
+	})
+
+	t.Run("with envoy", func(t *testing.T) {
+		fakeEnvoyMetricsServer := httptest.NewServer(newEnvoyMetricsHandler())
+		envoyURL = fakeEnvoyMetricsServer.URL
+		b := getMetrics(t)
+
+		if m, _ := regexp.Match(`(?m)^go_.*`, b); !m {
+			t.Errorf("Metrics endpoint did not contain internal metrics: %s", b)
+		}
+		if m, _ := regexp.Match(`(?m)^# TYPE envoy_.*`, b); !m {
+			t.Errorf("Metrics endpoint did not contain envoy metrics: %s", b)
+		}
+
+	})
+
 }