Crd autodiscovery improvements

config/crd/getport.io_crdsyncers.yaml

@@ -0,0 +1,99 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.0
+  name: crdsyncers.getport.io
+spec:
+  group: getport.io
+  names:
+    kind: CrdSyncer
+    listKind: CrdSyncerList
+    plural: crdsyncers
+    singular: crdsyncer
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              discover:
+                items:
+                  properties:
+                    automation:
+                      properties:
+                        invocationMethod:
+                          properties:
+                            org:
+                              type: string
+                            repo:
+                              type: string
+                            reportWorkflowStatus:
+                              type: boolean
+                            type:
+                              type: string
+                            workflow:
+                              type: string
+                          required:
+                          - org
+                          - repo
+                          - reportWorkflowStatus
+                          - type
+                          - workflow
+                          type: object
+                      required:
+                      - invocationMethod
+                      type: object
+                    filter:
+                      type: string
+                    propertiesAllowJQPattern:
+                      type: string
+                    propertiesAllowList:
+                      items:
+                        type: string
+                      type: array
+                    propertiesDenyJQPattern:
+                      type: string
+                    propertiesDenyList:
+                      items:
+                        type: string
+                      type: array
+                  required:
+                  - filter
+                  type: object
+                type: array
+            required:
+            - discover
+            type: object
+          status:
+            properties:
+              conditions:
+                items:
+                  type: string
+                type: array
+            required:
+            - conditions
+            type: object
+        type: object
+    served: true
+    storage: true

hack/gen-types.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# This script generates the CRDs from the types of the k8s exporter
+
+echo 'Installing required tools'
+
+go install sigs.k8s.io/controller-tools/cmd/controller-gen@latest
+
+echo 'Generating types...'
+
+controller-gen paths="./pkg/api/..." crd 
+
+if [ $? -ne 0 ]; then
+    echo 'Error generating types'
+    exit 1
+else
+    echo 'Types generated successfully'
+fi

pkg/api/v1alpha1/crd_syncer_types.go

@@ -0,0 +1,56 @@
+// +groupName=getport.io
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	GroupVersion = schema.GroupVersion{Group: "getport.io", Version: "v1alpha1"}
+)
+
+type InvocationMethod struct {
+	Type                 string `json:"type"`
+	Org                  string `json:"org"`
+	Repo                 string `json:"repo"`
+	Workflow             string `json:"workflow"`
+	ReportWorkflowStatus bool   `json:"reportWorkflowStatus"`
+}
+
+type Automation struct {
+	InvocationMethod InvocationMethod `json:"invocationMethod"`
+}
+
+type Discover struct {
+	Filter                   string     `json:"filter"`
+	PropertiesDenyList       []string   `json:"propertiesDenyList,omitempty"`
+	PropertiesAllowList      []string   `json:"propertiesAllowList,omitempty"`
+	PropertiesAllowJQPattern string     `json:"propertiesAllowJQPattern,omitempty"`
+	PropertiesDenyJQPattern  string     `json:"propertiesDenyJQPattern,omitempty"`
+	Automation               Automation `json:"automation,omitempty"`
+}
+
+type CrdSyncerSpec struct {
+	Discover []Discover `json:"discover"`
+}
+
+type Status struct {
+	Conditions []string `json:"conditions"`
+}
+
+type CrdSyncer struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   CrdSyncerSpec `json:"spec,omitempty"`
+	Status Status        `json:"status,omitempty"`
+}
+
+type CrdSyncerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []CrdSyncer `json:"items"`
+}

pkg/crd_syncer/controllers.go

@@ -0,0 +1,96 @@
+package crdsyncer
+
+import (
+	"github.com/port-labs/port-k8s-exporter/pkg/port"
+	"github.com/port-labs/port-k8s-exporter/pkg/port/cli"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+
+	crdSyncerV1alpha1 "github.com/port-labs/port-k8s-exporter/pkg/api/v1alpha1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+)
+
+type EventActionType string
+
+const (
+	CreateAction   EventActionType = "create"
+	UpdateAction   EventActionType = "update"
+	DeleteAction   EventActionType = "delete"
+	MaxNumRequeues int             = 4
+)
+
+type EventItem struct {
+	Key        string
+	ActionType EventActionType
+}
+
+type Controller struct {
+	portClient *cli.PortClient
+	informer   cache.SharedIndexInformer
+	workqueue  workqueue.RateLimitingInterface
+}
+
+func InitalizeCRDSyncerControllers(portClient *cli.PortClient, portConfig *port.IntegrationAppConfig, apiExtensionsClient apiextensions.ApiextensionsV1Interface, informersFactory dynamicinformer.DynamicSharedInformerFactory) {
+	crdInformer := informersFactory.ForResource(schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"})
+	syncerConfigurationInformer := informersFactory.ForResource(schema.GroupVersionResource{Group: "getport.io", Version: "v1", Resource: "crdsyncers"})
+
+	crdController := &Controller{
+		portClient: portClient,
+		informer:   crdInformer.Informer(),
+		workqueue:  workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
+	}
+
+	// crdController.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+	// If a CRD is added we want to see wheter he matches any of the CRD Syncers and if so to add it
+	// AddFunc: func(obj interface{}) {
+	// var err error
+	// var item EventItem
+	// item.ActionType = CreateAction
+	// item.Key, err = cache.MetaNamespaceKeyFunc(obj)
+	// if err == nil {
+	// AutoDiscoverSingleCRDToAction(portConfig, apiExtensionsClient, portClient, item.Key)
+	// }
+	// },
+	// If a CRD changed we want to update the relevant blueprint in Port (if there's a mapping that matches the CRD)
+	// UpdateFunc: func(old interface{}, new interface{}) {
+	// var err error
+	// var item EventItem
+	// item.ActionType = UpdateAction
+	// item.Key, err = cache.MetaNamespaceKeyFunc(new)
+	// if err == nil {
+	// AutoDiscoverSingleCRDToAction(portConfig, apiExtensionsClient, portClient, item.Key)
+	// }
+	// },
+	// })
+
+	syncerConfigurationInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		// If there's a new file of CRDSyncer configuration we want to see if there's a CRD that matches it and if so to add it
+		AddFunc: func(obj interface{}) {
+			if crdSyncerConfiguration, ok := obj.(*crdSyncerV1alpha1.CrdSyncer); ok {
+				var err error
+				var item EventItem
+				item.ActionType = CreateAction
+				item.Key, err = cache.MetaNamespaceKeyFunc(obj)
+				if err == nil {
+					Sync([]crdSyncerV1alpha1.CrdSyncer{*crdSyncerConfiguration}, apiExtensionsClient, portClient, item.Key)
+				}
+			}
+
+		},
+		// If there's an update to the CRDSyncer configuration we want to update the relevant blueprints/automations that affected from it in Port
+		UpdateFunc: func(old interface{}, new interface{}) {
+			if crdSyncerConfiguration, ok := new.(*crdSyncerV1alpha1.CrdSyncer); ok {
+				var err error
+				var item EventItem
+
+				item.ActionType = UpdateAction
+				item.Key, err = cache.MetaNamespaceKeyFunc(new)
+				if err == nil {
+					Sync([]crdSyncerV1alpha1.CrdSyncer{*crdSyncerConfiguration}, apiExtensionsClient, portClient, item.Key)
+				}
+			}
+		},
+	})
+}

pkg/crd/crd_test.go → pkg/crd_syncer/crd_test.go

@@ -1,4 +1,4 @@
-package crd
+package crdsyncer
 
 import (
 	"slices"
@@ -263,7 +263,7 @@ func checkBlueprintAndActionsProperties(t *testing.T, f *Fixture, namespaced boo
 func TestCRD_crd_autoDiscoverCRDsToActionsClusterScoped(t *testing.T) {
 	f := newFixture(t, "", false, "true")
 
-	AutodiscoverCRDsToActions(f.portConfig, f.apiextensionClient, f.portClient)
+	AutoDiscoverSingleCRDToAction(f.portConfig, f.apiextensionClient, f.portClient, "testkind")
 
 	checkBlueprintAndActionsProperties(t, f, false)
 
@@ -273,7 +273,7 @@ func TestCRD_crd_autoDiscoverCRDsToActionsClusterScoped(t *testing.T) {
 func TestCRD_crd_autoDiscoverCRDsToActionsNamespaced(t *testing.T) {
 	f := newFixture(t, "", true, "true")
 
-	AutodiscoverCRDsToActions(f.portConfig, f.apiextensionClient, f.portClient)
+	AutoDiscoverSingleCRDToAction(f.portConfig, f.apiextensionClient, f.portClient, "testkind")
 
 	checkBlueprintAndActionsProperties(t, f, true)
 
@@ -283,7 +283,7 @@ func TestCRD_crd_autoDiscoverCRDsToActionsNamespaced(t *testing.T) {
 func TestCRD_crd_autoDiscoverCRDsToActionsNoCRDs(t *testing.T) {
 	f := newFixture(t, "", false, "false")
 
-	AutodiscoverCRDsToActions(f.portConfig, f.apiextensionClient, f.portClient)
+	AutoDiscoverSingleCRDToAction(f.portConfig, f.apiextensionClient, f.portClient, "testkind")
 
 	testUtils.CheckResourcesExistence(false, f.portClient, t, []string{"testkind"}, []string{}, []string{"create_testkind", "update_testkind", "delete_testkind"})
 }

pkg/crd/crd.go → pkg/crd_syncer/sync.go

@@ -1,11 +1,12 @@
-package crd
+package crdsyncer
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
 
+	crdSyncerV1alpha1 "github.com/port-labs/port-k8s-exporter/pkg/api/v1alpha1"
 	"github.com/port-labs/port-k8s-exporter/pkg/goutils"
 	"github.com/port-labs/port-k8s-exporter/pkg/jq"
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
@@ -348,13 +349,9 @@
 			_, err = cli.CreateAction(portClient, act)
 			if err != nil {
 				if strings.Contains(err.Error(), "taken") {
-					if portConfig.OverwriteCRDsActions {
-						_, err = cli.UpdateAction(portClient, act)
-						if err != nil {
-							klog.Errorf("Error updating blueprint action: %s", err.Error())
-						}
-					} else {
-						klog.Infof("Action already exists, if you wish to overwrite it, delete it first or provide the configuration overwriteCrdsActions: true, in the exporter configuration and resync")
+					_, err = cli.UpdateAction(portClient, act)
+					if err != nil {
+						klog.Errorf("Error updating blueprint action: %s", err.Error())
 					}
 				} else {
 					klog.Errorf("Error creating blueprint action: %s", err.Error())
@@ -364,19 +361,15 @@
 	}
 }
 
-func AutodiscoverCRDsToActions(portConfig *port.IntegrationAppConfig, apiExtensionsClient apiextensions.ApiextensionsV1Interface, portClient *cli.PortClient) {
-	if portConfig.CRDSToDiscover == "" {
-		klog.Info("Discovering CRDs is disabled")
-		return
-	}
-
-	klog.Infof("Discovering CRDs/XRDs with pattern: %s", portConfig.CRDSToDiscover)
-	crds, err := apiExtensionsClient.CustomResourceDefinitions().List(context.Background(), metav1.ListOptions{})
+func Sync(configurations []crdSyncerV1alpha1.CrdSyncer, apiExtensionsClient apiextensions.ApiextensionsV1Interface, portClient *cli.PortClient, crdName string) {
+	// Get all the CRDS
+	klog.Infof("Trying to export CRD/XRD: %s", crdName)
+	crd, err := apiExtensionsClient.CustomResourceDefinitions().Get(context.Background(), crdName, metav1.GetOptions{})
 
 	if err != nil {
-		klog.Errorf("Error listing CRDs: %s", err.Error())
+		klog.Errorf("Error getting CRD: %s", err.Error())
 		return
 	}
 
-	handleCRD(crds.Items, portConfig, portClient)
+	handleCRD([]v1.CustomResourceDefinition{*crd}, portConfig, portClient)
 }

pkg/crd/utils.go → pkg/crd_syncer/utils.go

@@ -1,4 +1,4 @@
-package crd
+package crdsyncer
 
 import (
 	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"

pkg/crd/utils_test.go → pkg/crd_syncer/utils_test.go

@@ -1,4 +1,4 @@
-package crd
+package crdsyncer
 
 import (
 	"reflect"

pkg/handlers/controllers.go

@@ -5,9 +5,9 @@ import (
 	"time"
 
 	"github.com/port-labs/port-k8s-exporter/pkg/config"
+	crdsyncer "github.com/port-labs/port-k8s-exporter/pkg/crd_syncer"
 	"github.com/port-labs/port-k8s-exporter/pkg/port/integration"
 
-	"github.com/port-labs/port-k8s-exporter/pkg/crd"
 	"github.com/port-labs/port-k8s-exporter/pkg/goutils"
 	"github.com/port-labs/port-k8s-exporter/pkg/k8s"
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
@@ -30,7 +30,9 @@ func NewControllersHandler(exporterConfig *port.Config, portConfig *port.Integra
 	resync := time.Minute * time.Duration(exporterConfig.ResyncInterval)
 	informersFactory := dynamicinformer.NewDynamicSharedInformerFactory(k8sClient.DynamicClient, resync)
 
-	crd.AutodiscoverCRDsToActions(portConfig, k8sClient.ApiExtensionClient, portClient)
+	if portConfig.CRDSToDiscover != "" {
+		crdsyncer.InitalizeCRDSyncerControllers(portClient, portConfig, k8sClient.ApiExtensionClient, informersFactory)
+	}
 
 	aggResources := make(map[string][]port.KindConfig)
 	for _, resource := range portConfig.Resources {