Merge pull request #38 from port-labs/PORT-6197-k-8-s-exporter-add-ba…

…ckward-compatability-for-config-map

added OverwriteConfigurationOnRestart configuration

pkg/config/config.go

@@ -1,13 +1,13 @@
 package config
 
 import (
-	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"github.com/joho/godotenv"
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
+	"gopkg.in/yaml.v2"
 	"k8s.io/klog/v2"
+	"os"
 	"strings"
 )
 
@@ -37,6 +37,7 @@ func Init() {
 	NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.")
 	NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.")
 	NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.")
+	NewBool(&ApplicationConfig.OverwriteConfigurationOnRestart, "overwrite-configuration-on-restart", false, "Overwrite the configuration in port on restarting the exporter. Optional.")
 
 	// Deprecated
 	NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.")
@@ -46,33 +47,29 @@ func Init() {
 }
 
 func NewConfiguration() (*port.Config, error) {
-	overrides := &port.Config{
-		StateKey:                     ApplicationConfig.StateKey,
-		EventListenerType:            ApplicationConfig.EventListenerType,
-		CreateDefaultResources:       ApplicationConfig.CreateDefaultResources,
-		ResyncInterval:               ApplicationConfig.ResyncInterval,
-		CreateMissingRelatedEntities: ApplicationConfig.CreateMissingRelatedEntities,
-		DeleteDependents:             ApplicationConfig.DeleteDependents,
+	config := &port.Config{
+		StateKey:                        ApplicationConfig.StateKey,
+		EventListenerType:               ApplicationConfig.EventListenerType,
+		CreateDefaultResources:          ApplicationConfig.CreateDefaultResources,
+		ResyncInterval:                  ApplicationConfig.ResyncInterval,
+		OverwriteConfigurationOnRestart: ApplicationConfig.OverwriteConfigurationOnRestart,
+		CreateMissingRelatedEntities:    ApplicationConfig.CreateMissingRelatedEntities,
+		DeleteDependents:                ApplicationConfig.DeleteDependents,
 	}
 
-	c, err := GetConfigFile(ApplicationConfig.ConfigFilePath)
-	var fileNotFoundError *FileNotFoundError
-	if errors.As(err, &fileNotFoundError) {
+	v, err := os.ReadFile(ApplicationConfig.ConfigFilePath)
+	if err != nil {
+		v = []byte("{}")
 		klog.Infof("Config file not found, using defaults")
-		return overrides, nil
+		return config, nil
 	}
 	klog.Infof("Config file found")
-	v, err := json.Marshal(overrides)
-	if err != nil {
-		return nil, fmt.Errorf("failed loading configuration: %w", err)
-	}
-
-	err = json.Unmarshal(v, &c)
+	err = yaml.Unmarshal(v, &config)
 	if err != nil {
 		return nil, fmt.Errorf("failed loading configuration: %w", err)
 	}
 
-	c.StateKey = strings.ToLower(c.StateKey)
+	config.StateKey = strings.ToLower(config.StateKey)
 
-	return c, nil
+	return config, nil
 }

pkg/config/models.go

@@ -13,18 +13,17 @@ type KafkaConfiguration struct {
 }
 
 type ApplicationConfiguration struct {
-	ConfigFilePath         string
-	StateKey               string
-	ResyncInterval         uint
-	PortBaseURL            string
-	PortClientId           string
-	PortClientSecret       string
-	EventListenerType      string
-	CreateDefaultResources bool
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
-	Resources []port.Resource
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
-	DeleteDependents bool `json:"deleteDependents,omitempty"`
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
+	ConfigFilePath                  string
+	StateKey                        string
+	ResyncInterval                  uint
+	PortBaseURL                     string
+	PortClientId                    string
+	PortClientSecret                string
+	EventListenerType               string
+	CreateDefaultResources          bool
+	OverwriteConfigurationOnRestart bool
+	// These Configurations are used only for setting up the Integration on installation or when using OverwriteConfigurationOnRestart flag.
+	Resources                    []port.Resource
+	DeleteDependents             bool `json:"deleteDependents,omitempty"`
 	CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"`
 }

pkg/config/utils.go

@@ -3,10 +3,7 @@ package config
 import (
 	"flag"
 	"github.com/port-labs/port-k8s-exporter/pkg/goutils"
-	"github.com/port-labs/port-k8s-exporter/pkg/port"
-	"gopkg.in/yaml.v2"
 	"k8s.io/utils/strings/slices"
-	"os"
 	"strings"
 )
 
@@ -37,26 +34,3 @@ func NewBool(v *bool, key string, defaultValue bool, description string) {
 	value := goutils.GetBoolEnvOrDefault(prepareEnvKey(key), defaultValue)
 	flag.BoolVar(v, key, value, description)
 }
-
-type FileNotFoundError struct {
-	s string
-}
-
-func (e *FileNotFoundError) Error() string {
-	return e.s
-}
-
-func GetConfigFile(filepath string) (*port.Config, error) {
-	c := &port.Config{}
-	config, err := os.ReadFile(filepath)
-	if err != nil {
-		return c, &FileNotFoundError{err.Error()}
-	}
-
-	err = yaml.Unmarshal(config, c)
-	if err != nil {
-		return nil, err
-	}
-
-	return c, nil
-}

pkg/defaults/defaults.go

@@ -144,19 +144,22 @@ func validateResourcesErrors(createdBlueprints []string, createdPages []string,
 func validateResourcesDoesNotExist(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError {
 	var errors []error
 	if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil {
+		klog.Warningf("Integration with state key %s already exists", config.StateKey)
 		return &AbortDefaultCreationError{Errors: []error{
 			fmt.Errorf("integration with state key %s already exists", config.StateKey),
 		}}
 	}
 
 	for _, bp := range defaults.Blueprints {
 		if _, err := blueprint.GetBlueprint(portClient, bp.Identifier); err == nil {
+			klog.Warningf("Blueprint with identifier %s already exists", bp.Identifier)
 			errors = append(errors, fmt.Errorf("blueprint with identifier %s already exists", bp.Identifier))
 		}
 	}
 
 	for _, p := range defaults.Pages {
 		if _, err := page.GetPage(portClient, p.Identifier); err == nil {
+			klog.Warningf("Page with identifier %s already exists", p.Identifier)
 			errors = append(errors, fmt.Errorf("page with identifier %s already exists", p.Identifier))
 		}
 	}
@@ -169,6 +172,7 @@ func validateResourcesDoesNotExist(portClient *cli.PortClient, defaults *Default
 
 func createResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError {
 	if err := validateResourcesDoesNotExist(portClient, defaults, config); err != nil {
+		klog.Warningf("Failed to create resources: %v.", err.Errors)
 		return err
 	}
 
@@ -189,8 +193,10 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por
 
 			mutex.Lock()
 			if err != nil {
+				klog.Warningf("Failed to create blueprint %s: %v", bp.Identifier, err.Error())
 				resourceErrors = append(resourceErrors, err)
 			} else {
+				klog.Infof("Created blueprint %s", result.Identifier)
 				createdBlueprints = append(createdBlueprints, result.Identifier)
 			}
 			mutex.Unlock()
@@ -208,6 +214,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por
 			go func(bp port.Blueprint) {
 				defer waitGroup.Done()
 				if _, err := blueprint.PatchBlueprint(portClient, bp); err != nil {
+					klog.Warningf("Failed to patch blueprint %s: %v", bp.Identifier, err.Error())
 					resourceErrors = append(resourceErrors, err)
 				}
 			}(bp)
@@ -225,6 +232,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por
 			go func(blueprintIdentifier string, scorecard port.Scorecard) {
 				defer waitGroup.Done()
 				if err := scorecards.CreateScorecard(portClient, blueprintIdentifier, scorecard); err != nil {
+					klog.Warningf("Failed to create scorecard for blueprint %s: %v", blueprintIdentifier, err.Error())
 					resourceErrors = append(resourceErrors, err)
 				}
 			}(blueprintScorecards.Blueprint, scorecard)
@@ -241,8 +249,10 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por
 		go func(p port.Page) {
 			defer waitGroup.Done()
 			if err := page.CreatePage(portClient, p); err != nil {
+				klog.Warningf("Failed to create page %s: %v", p.Identifier, err.Error())
 				resourceErrors = append(resourceErrors, err)
 			} else {
+				klog.Infof("Created page %s", p.Identifier)
 				createdPages = append(createdPages, p.Identifier)
 			}
 		}(pageToCreate)
@@ -254,7 +264,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por
 	}
 
 	if err := integration.CreateIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil {
-		klog.Infof("Failed to create resources: %v.", err.Error())
+		klog.Warningf("Failed to create integration with default configuration. state key %s: %v", config.StateKey, err.Error())
 		return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, PagesToRollback: createdPages, Errors: []error{err}}
 	}
 

pkg/defaults/defaults_test.go

@@ -192,7 +192,7 @@ func Test_InitIntegration_ExistingIntegration(t *testing.T) {
 	checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"})
 }
 
-func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) {
+func Test_InitIntegration_LocalResourcesConfiguration(t *testing.T) {
 	f := NewFixture(t)
 	err := integration.CreateIntegration(f.portClient, f.stateKey, "", nil)
 	if err != nil {
@@ -233,7 +233,7 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) {
 	checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"})
 }
 
-func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) {
+func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) {
 	f := NewFixture(t)
 	err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", nil)
 	if err != nil {
@@ -253,3 +253,49 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_E
 
 	checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"})
 }
+
+func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_WithConfiguration_WithOverwriteConfigurationOnRestartFlag(t *testing.T) {
+	f := NewFixture(t)
+	expectedConfig := &port.IntegrationAppConfig{
+		Resources: []port.Resource{
+			{
+				Kind: "workload",
+				Port: port.Port{
+					Entity: port.EntityMappings{
+						Mappings: []port.EntityMapping{
+							{
+								Identifier: "\"workload\"",
+								Title:      "\"Workload\"",
+								Blueprint:  "\"workload\"",
+								Icon:       "\"Microservice\"",
+								Properties: map[string]string{
+									"namespace": "\"default\"",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", expectedConfig)
+	if err != nil {
+		t.Errorf("Error creating Port integration: %s", err.Error())
+	}
+
+	expectedConfig.Resources[0].Kind = "namespace"
+	e := InitIntegration(f.portClient, &port.Config{
+		StateKey:                        f.stateKey,
+		EventListenerType:               "KAFKA",
+		Resources:                       expectedConfig.Resources,
+		CreateDefaultResources:          true,
+		OverwriteConfigurationOnRestart: true,
+	})
+	assert.Nil(t, e)
+
+	i, err := integration.GetIntegration(f.portClient, f.stateKey)
+	assert.Nil(t, err)
+	assert.Equal(t, expectedConfig.Resources, i.Config.Resources)
+
+	checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"})
+}

pkg/defaults/init.go

@@ -54,7 +54,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config)
 
 		// Handle a deprecated case where resources are provided in config file and integration exists from previous
 		//versions without a config
-		if existingIntegration.Config == nil {
+		if existingIntegration.Config == nil || applicationConfig.OverwriteConfigurationOnRestart {
 			klog.Infof("Integration exists without a config, patching it with default config: %v", defaultIntegrationConfig)
 			integrationPatch.Config = defaultIntegrationConfig
 		}

pkg/port/models.go

@@ -182,7 +182,7 @@ type EntityMapping struct {
 	Identifier string            `json:"identifier"`
 	Title      string            `json:"title"`
 	Blueprint  string            `json:"blueprint"`
-	Icon	   string            `json:"icon,omitempty"`
+	Icon       string            `json:"icon,omitempty"`
 	Team       string            `json:"team,omitempty"`
 	Properties map[string]string `json:"properties,omitempty"`
 	Relations  map[string]string `json:"relations,omitempty"`
@@ -227,14 +227,13 @@ type IntegrationAppConfig struct {
 }
 
 type Config struct {
-	ResyncInterval         uint
-	StateKey               string
-	EventListenerType      string
-	CreateDefaultResources bool
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
-	Resources []Resource `json:"resources,omitempty"`
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
-	DeleteDependents bool `json:"deleteDependents,omitempty"`
-	// Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup.
-	CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"`
+	ResyncInterval                  uint   `yaml:"resyncInterval,omitempty"`
+	StateKey                        string `yaml:"stateKey,omitempty"`
+	EventListenerType               string `yaml:"eventListenerType,omitempty"`
+	CreateDefaultResources          bool   `yaml:"createDefaultResources,omitempty"`
+	OverwriteConfigurationOnRestart bool   `yaml:"overwriteConfigurationOnRestart,omitempty"`
+	// These Configurations are used only for setting up the Integration on installation or when using OverwriteConfigurationOnRestart flag.
+	Resources                    []Resource `yaml:"resources,omitempty"`
+	DeleteDependents             bool       `yaml:"deleteDependents,omitempty"`
+	CreateMissingRelatedEntities bool       `yaml:"createMissingRelatedEntities,omitempty"`
 }