Fixed service-mirror metrics warning (#11246)

Whenever the service mirror's main loop was triggered again, the following warnings were generated:

```
time="2023-08-14T20:16:29Z" level=warning msg="failed to register Prometheus gauge Desc{fqName: \"service_cache_size\", help: \"Number of items in the client-go service cache\", constLabels: {cluster=\"remote\"}, variableLabels: []}: duplicate metrics collector registration attempted"
time="2023-08-14T20:16:29Z" level=warning msg="failed to register Prometheus gauge Desc{fqName: \"endpoints_cache_size\", help: \"Number of items in the client-go endpoints cache\", constLabels: {cluster=\"remote\"}, variableLabels: []}: duplicate metrics collector registration attempted"
```

To fix, this adds into the cluster watcher's `Stop()` method a directive to unregister the prometheus cache metrics associated to the cluster's client API.

controller/k8s/api.go

@@ -288,6 +288,11 @@ func (api *API) Sync(stopCh <-chan struct{}) {
 	waitForCacheSync(api.syncChecks)
 }
 
+// UnregisterGauges unregisters all the prometheus cache gauges associated to this API
+func (api *API) UnregisterGauges() {
+	api.promGauges.unregister()
+}
+
 // NS provides access to a shared informer and lister for Namespaces.
 func (api *API) NS() coreinformers.NamespaceInformer {
 	if api.ns == nil {

controller/k8s/prometheus.go

@@ -20,3 +20,9 @@ func (p *promGauges) addInformerSize(kind string, labels prometheus.Labels, inf
 		return float64(len(inf.GetStore().ListKeys()))
 	}))
 }
+
+func (p *promGauges) unregister() {
+	for _, gauge := range p.gauges {
+		prometheus.Unregister(gauge)
+	}
+}

multicluster/service-mirror/cluster_watcher.go

@@ -974,6 +974,10 @@ func (rcsw *RemoteClusterServiceWatcher) Stop(cleanupState bool) {
 			rcsw.log.Warnf("error removing service informer handler: %s", err)
 		}
 	}
+
+	if rcsw.remoteAPIClient != nil {
+		rcsw.remoteAPIClient.UnregisterGauges()
+	}
 }
 
 func (rcsw *RemoteClusterServiceWatcher) resolveGatewayAddress() ([]corev1.EndpointAddress, error) {