Remove linkerd-viz dependency from linkerd-mutlicluster gateways co…

…mmand (linkerd#8467)

This changes linkerd-multicluster's `gateways` command to use the service mirror component's `/metrics` endpoint so that there is no longer a dependency on linkerd-viz. The dependency on linkerd-viz is leftover from when those components were part of the default installation meaning that we could always rely on the Prometheus component being present.

Now, the `gateways` command starts a port-forward to each service mirror component (for each linked cluster) and queries the `/metrics` endpoint for the `gateway_alive` and `gateway_latency` metrics. It then queries the local cluster for the number of mirror services that correspond to the target cluster of that service mirror. Using these three data points, it creates the output table for the command.

### Output changes

Currently the `gateways` command displays the P50, P95, and P99 latencies for each gateway

```shell
$ linkerd multicluster gateways 
CLUSTER  ALIVE    NUM_SVC  LATENCY_P50  LATENCY_P95  LATENCY_P99  
k3d-x    True           1          1ms          3ms          3ms  
k3d-z    True           0          1ms          3ms          3ms
```

With this change, we now just show the last observed latency. This involved adding the `gateway_latency` metric Gauge — different from the current latencies Observer.

```shell
$ linkerd multicluster gateways
CLUSTER  ALIVE    NUM_SVC      LATENCY  
k3d-x    True           1          2ms  
k3d-z    True           0          3ms
```

This is because I have not found a Prometheus common library for taking the parsed metrics from `/metrics` and turning that into a histogram yet; I think we should be able to do this but I'm leaving as a follow-up for now.

Signed-off-by: Kevin Leimkuhler <[email protected]>

cli/cmd/controller-metrics.go

@@ -50,7 +50,7 @@ func newCmdControllerMetrics() *cobra.Command {
 				return err
 			}
 
-			results := getMetrics(k8sAPI, pods.Items, adminHTTPPortName, options.wait, verbose)
+			results := getMetrics(k8sAPI, pods.Items, k8s.AdminHTTPPortName, options.wait, verbose)
 
 			var buf bytes.Buffer
 			for i, result := range results {

cli/cmd/diagnostics.go

@@ -4,10 +4,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const (
-	adminHTTPPortName string = "admin-http"
-)
-
 // newCmdDiagnostics creates a new cobra command `diagnostics` which contains commands to fetch Linkerd diagnostics
 func newCmdDiagnostics() *cobra.Command {
 
@@ -24,7 +20,7 @@ This command provides subcommands to diagnose the functionality of Linkerd.`,
 
   # Get metrics from the web deployment in the emojivoto namespace.
   linkerd diagnostics proxy-metrics -n emojivoto deploy/web
- 
+
   # Get the endpoints for authorities in Linkerd's control-plane itself
   linkerd diagnostics endpoints web.linkerd-viz.svc.cluster.local:8084
   `,

cli/cmd/metrics_diagnostics_util.go

@@ -4,9 +4,6 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
 	"sort"
 	"sync/atomic"
 	"time"
@@ -35,42 +32,6 @@ func (s byResult) Less(i, j int) bool {
 	return s[i].pod < s[j].pod || ((s[i].pod == s[j].pod) && s[i].container < s[j].container)
 }
 
-// getResponse makes a http Get request to the passed url and returns the response/error
-func getResponse(url string) ([]byte, error) {
-	// url has been constructed by k8s.newPortForward and is not passed in by
-	// the user.
-	//nolint:gosec
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	return ioutil.ReadAll(resp.Body)
-}
-
-// getContainerMetrics returns the metrics exposed by a container on the passed in portName
-func getContainerMetrics(
-	k8sAPI *k8s.KubernetesAPI,
-	pod corev1.Pod,
-	container corev1.Container,
-	emitLogs bool,
-	portName string,
-) ([]byte, error) {
-	portForward, err := k8s.NewContainerMetricsForward(k8sAPI, pod, container, emitLogs, portName)
-	if err != nil {
-		return nil, err
-	}
-
-	defer portForward.Stop()
-	if err = portForward.Init(); err != nil {
-		fmt.Fprintf(os.Stderr, "Error running port-forward: %s", err)
-		return nil, err
-	}
-
-	metricsURL := portForward.URLFor("/metrics")
-	return getResponse(metricsURL)
-}
-
 // getAllContainersWithPort returns all the containers within
 // a pod which exposes metrics at a port with name portName
 func getAllContainersWithPort(
@@ -119,7 +80,7 @@ func getMetrics(
 			}
 
 			for _, c := range containers {
-				bytes, err := getContainerMetrics(k8sAPI, p, c, emitLogs, portName)
+				bytes, err := k8s.GetContainerMetrics(k8sAPI, p, c, emitLogs, portName)
 
 				resultChan <- metricsResult{
 					pod:       p.GetName(),