Admin endpoint: allow overriding kube gateway gvks (#9716)

* extensions startfuncs

* add convert code

* make getRedactedApiSnapshot private

* flatten kube resources

* rename client

* fix tests

* changelog

* add comments and test

* return more resources

* refactor gvks check

* refactor numfields check

* updates

* set gvks

* pass history as param

* sort proxies too

* remove the startfuncs stuff

* changelog

* add more tests

* update api

* changelog

* add tests

* refactor sort

* Adding changelog file to new location

* Deleting changelog file from old location

* refactor

* fix tests

* fix compile

* empty

* add SnapshotHistoryFactory to validation

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

changelog/v1.18.0-beta9/admin-endpoint-gvks.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6391
+    resolvesIssue: false
+    description: >-
+      Allow the list of Kubernetes Gateway-related resource types returned by the Admin input snapshot endpoint
+      to be configured via the History object.

projects/gateway2/controller/start.go

@@ -3,14 +3,6 @@ package controller
 import (
 	"context"
 
-	"github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot"
-
-	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/healthz"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
-
 	gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
 	"github.com/solo-io/gloo/projects/gateway2/controller/scheme"
 	"github.com/solo-io/gloo/projects/gateway2/extensions"
@@ -21,7 +13,13 @@ import (
 	api "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1"
 	"github.com/solo-io/gloo/projects/gloo/pkg/bootstrap"
 	"github.com/solo-io/gloo/projects/gloo/pkg/plugins"
+	"github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot"
 	"github.com/solo-io/solo-kit/pkg/api/v2/reporter"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 const (

projects/gateway2/extensions/extensions.go

@@ -3,11 +3,10 @@ package extensions
 import (
 	"context"
 
-	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1"
-
 	gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
 	"github.com/solo-io/gloo/projects/gateway2/query"
 	"github.com/solo-io/gloo/projects/gateway2/translator/plugins/registry"
+	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1"
 	"github.com/solo-io/solo-kit/pkg/api/v2/reporter"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 )

projects/gloo/pkg/servers/iosnapshot/convert.go

@@ -1,9 +1,6 @@
 package iosnapshot
 
 import (
-	"cmp"
-	"slices"
-
 	gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
 	ratelimitv1alpha1 "github.com/solo-io/gloo/projects/gloo/pkg/api/external/solo/ratelimit"
 	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
@@ -168,14 +165,3 @@ func convertToKube(resource resources.VersionedResource, crd crd.Crd) (*crdv1.Re
 		Status:     crdv1.Status{},
 	}, nil
 }
-
-func sortResources(resources []crdv1.Resource) {
-	slices.SortStableFunc(resources, func(a, b crdv1.Resource) int {
-		return cmp.Or(
-			cmp.Compare(a.APIVersion, b.APIVersion),
-			cmp.Compare(a.Kind, b.Kind),
-			cmp.Compare(a.GetNamespace(), b.GetNamespace()),
-			cmp.Compare(a.GetName(), b.GetName()),
-		)
-	})
-}

projects/gloo/pkg/servers/iosnapshot/format.go

@@ -1,10 +1,21 @@
 package iosnapshot
 
 import (
+	"cmp"
 	"encoding/json"
 	"fmt"
+	"slices"
+
+	crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1"
 )
 
+// formatResources sorts the resources and formats them into json output
+func formatResources(resources []crdv1.Resource) ([]byte, error) {
+	sortResources(resources)
+	return formatOutput("json_compact", resources)
+}
+
+// formatOutput formats a generic object into the specified output format
 func formatOutput(format string, genericOutput interface{}) ([]byte, error) {
 	switch format {
 	case "json":
@@ -18,5 +29,16 @@ func formatOutput(format string, genericOutput interface{}) ([]byte, error) {
 	default:
 		return nil, fmt.Errorf("invalid format of %s", format)
 	}
+}
 
+// sortResources sorts resources by gvk, namespace, and name
+func sortResources(resources []crdv1.Resource) {
+	slices.SortStableFunc(resources, func(a, b crdv1.Resource) int {
+		return cmp.Or(
+			cmp.Compare(a.APIVersion, b.APIVersion),
+			cmp.Compare(a.Kind, b.Kind),
+			cmp.Compare(a.GetNamespace(), b.GetNamespace()),
+			cmp.Compare(a.GetName(), b.GetName()),
+		)
+	})
 }

projects/gloo/pkg/servers/iosnapshot/history.go

@@ -4,12 +4,7 @@ import (
 	"context"
 	"sync"
 
-	gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
-	"github.com/solo-io/gloo/projects/gateway2/api/v1alpha1"
-	"github.com/solo-io/gloo/projects/gateway2/wellknown"
-	ratelimitv1alpha1 "github.com/solo-io/gloo/projects/gloo/pkg/api/external/solo/ratelimit"
 	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
-	extauthv1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1"
 	v1snap "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/gloosnapshot"
 	crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1"
 	"github.com/solo-io/solo-kit/pkg/api/v1/control-plane/cache"
@@ -23,28 +18,58 @@ import (
 // The ControlPlane will use the Setters to update the last known state,
 // and the Getters will be used by the Admin Server
 type History interface {
-	// SetApiSnapshot sets the latest ApiSnapshot
+	// SetApiSnapshot sets the latest Edge ApiSnapshot.
 	SetApiSnapshot(latestInput *v1snap.ApiSnapshot)
+
 	// SetKubeGatewayClient sets the client to use for Kubernetes CRUD operations when
-	// Kubernetes Gateway integration is enabled. If this is not set, then no Kubernetes
-	// Gateway resources will be returned from `GetInputSnapshot`.
+	// Kubernetes Gateway integration is enabled. If this is not set, then it is assumed
+	// that Kubernetes Gateway integration is not enabled, and no Kubernetes Gateway
+	// resources will be returned from `GetInputSnapshot`.
 	SetKubeGatewayClient(kubeGatewayClient client.Client)
-	// GetInputSnapshot gets the input snapshot for all components.
+
+	// GetInputSnapshot returns all resources in the Edge input snapshot, and if Kubernetes
+	// Gateway integration is enabled, it additionally returns all resources on the cluster
+	// with types specified by `kubeGvks`.
 	GetInputSnapshot(ctx context.Context) ([]byte, error)
+
 	// GetProxySnapshot returns the Proxies generated for all components.
 	GetProxySnapshot(ctx context.Context) ([]byte, error)
+
 	// GetXdsSnapshot returns the entire cache of xDS snapshots
 	// NOTE: This contains sensitive data, as it is the exact inputs that used by Envoy
 	GetXdsSnapshot(ctx context.Context) ([]byte, error)
 }
 
+// HistoryFactoryParameters are the inputs used to create a History object
+type HistoryFactoryParameters struct {
+	Settings *gloov1.Settings
+	Cache    cache.SnapshotCache
+}
+
+// HistoryFactory is a function that produces a History object
+type HistoryFactory func(params HistoryFactoryParameters) History
+
+// GetHistoryFactory returns a default HistoryFactory implementation
+func GetHistoryFactory() HistoryFactory {
+	return func(params HistoryFactoryParameters) History {
+		return NewHistory(params.Cache, params.Settings, KubeGatewayDefaultGVKs)
+	}
+}
+
 // NewHistory returns an implementation of the History interface
-func NewHistory(cache cache.SnapshotCache, settings *gloov1.Settings) History {
+//   - `cache` is the control plane's xDS snapshot cache
+//   - `settings` specifies the Settings for this control plane instance
+//   - `kubeGvks` specifies the list of resource types to return in the input snapshot when
+//     Kubernetes Gateway integration is enabled. For example, this may include Gateway API
+//     resources, Portal resources, or other resources specific to the Kubernetes Gateway integration.
+//     If not set, then only Edge ApiSnapshot resources will be returned from `GetInputSnapshot`.
+func NewHistory(cache cache.SnapshotCache, settings *gloov1.Settings, kubeGvks []schema.GroupVersionKind) History {
 	return &historyImpl{
 		latestApiSnapshot: nil,
 		xdsCache:          cache,
 		settings:          settings,
 		kubeGatewayClient: nil,
+		kubeGvks:          kubeGvks,
 	}
 }
 
@@ -57,6 +82,9 @@ type historyImpl struct {
 	xdsCache          cache.SnapshotCache
 	settings          *gloov1.Settings
 	kubeGatewayClient client.Client
+	// this will hold all the kube gvks that we want to show in the input snapshot when kube gateway
+	// integration is enabled
+	kubeGvks []schema.GroupVersionKind
 }
 
 // SetApiSnapshot sets the latest input ApiSnapshot
@@ -127,10 +155,7 @@ func (h *historyImpl) GetInputSnapshot(ctx context.Context) ([]byte, error) {
 	}
 	resources = append(resources, kubeResources...)
 
-	// sort all the resources before formatting
-	sortResources(resources)
-
-	return formatOutput("json_compact", resources)
+	return formatResources(resources)
 }
 
 func (h *historyImpl) GetProxySnapshot(ctx context.Context) ([]byte, error) {
@@ -144,7 +169,8 @@ func (h *historyImpl) GetProxySnapshot(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return formatOutput("json_compact", resources)
+
+	return formatResources(resources)
 }
 
 // GetXdsSnapshot returns the entire cache of xDS snapshots
@@ -213,25 +239,7 @@ func (h *historyImpl) getKubeGatewayResources(ctx context.Context) ([]crdv1.Reso
 	}
 
 	resources := []crdv1.Resource{}
-	gvks := []schema.GroupVersionKind{
-		// Kubernetes Gateway API resources
-		wellknown.GatewayClassListGVK,
-		wellknown.GatewayListGVK,
-		wellknown.HTTPRouteListGVK,
-		wellknown.ReferenceGrantListGVK,
-
-		// Gloo resources used in Kubernetes Gateway integration
-		v1alpha1.GatewayParametersGVK,
-		gatewayv1.ListenerOptionGVK,
-		gatewayv1.HttpListenerOptionGVK,
-
-		// resources shared between Edge and Kubernetes Gateway integration
-		gatewayv1.RouteOptionGVK,
-		gatewayv1.VirtualHostOptionGVK,
-		extauthv1.AuthConfigGVK,
-		ratelimitv1alpha1.RateLimitConfigGVK,
-	}
-	for _, gvk := range gvks {
+	for _, gvk := range h.kubeGvks {
 		// populate an unstructured list for each resource type
 		list := &unstructured.UnstructuredList{}
 		list.SetGroupVersionKind(gvk)
@@ -255,6 +263,7 @@ func (h *historyImpl) getKubeGatewayResources(ctx context.Context) ([]crdv1.Reso
 }
 
 // getKubeGatewayClientSafe gets the Kubernetes client used for CRUD operations
+// on resources used in the Kubernetes Gateway integration
 func (h *historyImpl) getKubeGatewayClientSafe() client.Client {
 	h.RLock()
 	defer h.RUnlock()

projects/gloo/pkg/servers/iosnapshot/history_test.go

@@ -31,6 +31,7 @@ import (
 	crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1"
 	"github.com/solo-io/solo-kit/pkg/api/v1/control-plane/cache"
 	"github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
+	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -65,12 +66,15 @@ var _ = Describe("History", func() {
 
 		clientBuilder = fake.NewClientBuilder().WithScheme(scheme)
 		xdsCache = &xds.MockXdsCache{}
-		history = NewHistory(xdsCache, &v1.Settings{
-			Metadata: &core.Metadata{
-				Name:      "my-settings",
-				Namespace: defaults.GlooSystem,
+		history = NewHistory(xdsCache,
+			&v1.Settings{
+				Metadata: &core.Metadata{
+					Name:      "my-settings",
+					Namespace: defaults.GlooSystem,
+				},
 			},
-		})
+			KubeGatewayDefaultGVKs,
+		)
 	})
 
 	Context("GetInputSnapshot", func() {
@@ -562,6 +566,78 @@ var _ = Describe("History", func() {
 				expectDoesNotContainResource(returnedResources, extauthv1.AuthConfigGVK, defaults.GlooSystem, "ac-snap")
 				expectDoesNotContainResource(returnedResources, ratelimitv1alpha1.RateLimitConfigGVK, defaults.GlooSystem, "rlc-snap")
 			})
+
+			It("returns only relevant gvks", func() {
+				clientObjects := []client.Object{
+					&appsv1.Deployment{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "kube-deploy",
+							Namespace: "a",
+						},
+					},
+				}
+
+				setClientOnHistory(ctx, history, clientBuilder.WithObjects(clientObjects...))
+
+				inputSnapshotBytes, err := history.GetInputSnapshot(ctx)
+				Expect(err).NotTo(HaveOccurred())
+
+				returnedResources := []crdv1.Resource{}
+				err = json.Unmarshal(inputSnapshotBytes, &returnedResources)
+				Expect(err).NotTo(HaveOccurred())
+
+				// a Deployment is not one of the resource types we return in the input snapshot by default, so
+				// the deployment should not appear in the results
+				deploymentGvk := schema.GroupVersionKind{
+					Group:   appsv1.GroupName,
+					Version: "v1",
+					Kind:    "Deployment",
+				}
+				Expect(containsResourceType(returnedResources, deploymentGvk)).To(BeFalse(),
+					"input snapshot should not contain deployments")
+			})
+
+			It("respects extra kube gvks", func() {
+				// create a new History that adds deployments to the kube input snapshot gvks
+				deploymentGvk := schema.GroupVersionKind{
+					Group:   appsv1.GroupName,
+					Version: "v1",
+					Kind:    "Deployment",
+				}
+				gvks := []schema.GroupVersionKind{}
+				gvks = append(gvks, KubeGatewayDefaultGVKs...)
+				gvks = append(gvks, deploymentGvk)
+				history = NewHistory(xdsCache,
+					&v1.Settings{
+						Metadata: &core.Metadata{
+							Name:      "my-settings",
+							Namespace: defaults.GlooSystem,
+						},
+					},
+					gvks,
+				)
+
+				// create a deployment
+				clientObjects := []client.Object{
+					&appsv1.Deployment{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "kube-deploy",
+							Namespace: "a",
+						},
+					},
+				}
+				setClientOnHistory(ctx, history, clientBuilder.WithObjects(clientObjects...))
+
+				inputSnapshotBytes, err := history.GetInputSnapshot(ctx)
+				Expect(err).NotTo(HaveOccurred())
+
+				returnedResources := []crdv1.Resource{}
+				err = json.Unmarshal(inputSnapshotBytes, &returnedResources)
+				Expect(err).NotTo(HaveOccurred())
+
+				// we should now see the deployment in the input snapshot results
+				expectContainsResource(returnedResources, deploymentGvk, "a", "kube-deploy")
+			})
 		})
 	})
 
@@ -616,7 +692,6 @@ func setSnapshotOnHistory(ctx context.Context, history History, snap *v1snap.Api
 // nature of the `Set` API on the History
 func setClientOnHistory(ctx context.Context, history History, builder *fake.ClientBuilder) {
 	gwSignalObject := &apiv1.Gateway{
-		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "gw-signal",
 			Namespace: defaults.GlooSystem,