diff --git a/pkg/config/config.go b/pkg/config/config.go index da6e369..625cff3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,13 +1,13 @@ package config import ( - "encoding/json" - "errors" "flag" "fmt" "github.com/joho/godotenv" "github.com/port-labs/port-k8s-exporter/pkg/port" + "gopkg.in/yaml.v2" "k8s.io/klog/v2" + "os" "strings" ) @@ -37,6 +37,7 @@ func Init() { NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") + NewBool(&ApplicationConfig.OverwriteConfigurationOnRestart, "overwrite-configuration-on-restart", false, "Overwrite the configuration in port on restarting the exporter. Optional.") // Deprecated NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.") @@ -46,33 +47,29 @@ func Init() { } func NewConfiguration() (*port.Config, error) { - overrides := &port.Config{ - StateKey: ApplicationConfig.StateKey, - EventListenerType: ApplicationConfig.EventListenerType, - CreateDefaultResources: ApplicationConfig.CreateDefaultResources, - ResyncInterval: ApplicationConfig.ResyncInterval, - CreateMissingRelatedEntities: ApplicationConfig.CreateMissingRelatedEntities, - DeleteDependents: ApplicationConfig.DeleteDependents, + config := &port.Config{ + StateKey: ApplicationConfig.StateKey, + EventListenerType: ApplicationConfig.EventListenerType, + CreateDefaultResources: ApplicationConfig.CreateDefaultResources, + ResyncInterval: ApplicationConfig.ResyncInterval, + OverwriteConfigurationOnRestart: ApplicationConfig.OverwriteConfigurationOnRestart, + CreateMissingRelatedEntities: ApplicationConfig.CreateMissingRelatedEntities, + DeleteDependents: ApplicationConfig.DeleteDependents, } - c, err := GetConfigFile(ApplicationConfig.ConfigFilePath) - var fileNotFoundError *FileNotFoundError - if errors.As(err, &fileNotFoundError) { + v, err := os.ReadFile(ApplicationConfig.ConfigFilePath) + if err != nil { + v = []byte("{}") klog.Infof("Config file not found, using defaults") - return overrides, nil + return config, nil } klog.Infof("Config file found") - v, err := json.Marshal(overrides) - if err != nil { - return nil, fmt.Errorf("failed loading configuration: %w", err) - } - - err = json.Unmarshal(v, &c) + err = yaml.Unmarshal(v, &config) if err != nil { return nil, fmt.Errorf("failed loading configuration: %w", err) } - c.StateKey = strings.ToLower(c.StateKey) + config.StateKey = strings.ToLower(config.StateKey) - return c, nil + return config, nil } diff --git a/pkg/config/models.go b/pkg/config/models.go index 4626e05..8020d39 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -13,18 +13,17 @@ type KafkaConfiguration struct { } type ApplicationConfiguration struct { - ConfigFilePath string - StateKey string - ResyncInterval uint - PortBaseURL string - PortClientId string - PortClientSecret string - EventListenerType string - CreateDefaultResources bool - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - Resources []port.Resource - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - DeleteDependents bool `json:"deleteDependents,omitempty"` - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + ConfigFilePath string + StateKey string + ResyncInterval uint + PortBaseURL string + PortClientId string + PortClientSecret string + EventListenerType string + CreateDefaultResources bool + OverwriteConfigurationOnRestart bool + // These Configurations are used only for setting up the Integration on installation or when using OverwriteConfigurationOnRestart flag. + Resources []port.Resource + DeleteDependents bool `json:"deleteDependents,omitempty"` CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` } diff --git a/pkg/config/utils.go b/pkg/config/utils.go index bf1f163..40c7d06 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -3,10 +3,7 @@ package config import ( "flag" "github.com/port-labs/port-k8s-exporter/pkg/goutils" - "github.com/port-labs/port-k8s-exporter/pkg/port" - "gopkg.in/yaml.v2" "k8s.io/utils/strings/slices" - "os" "strings" ) @@ -37,26 +34,3 @@ func NewBool(v *bool, key string, defaultValue bool, description string) { value := goutils.GetBoolEnvOrDefault(prepareEnvKey(key), defaultValue) flag.BoolVar(v, key, value, description) } - -type FileNotFoundError struct { - s string -} - -func (e *FileNotFoundError) Error() string { - return e.s -} - -func GetConfigFile(filepath string) (*port.Config, error) { - c := &port.Config{} - config, err := os.ReadFile(filepath) - if err != nil { - return c, &FileNotFoundError{err.Error()} - } - - err = yaml.Unmarshal(config, c) - if err != nil { - return nil, err - } - - return c, nil -} diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index f0644a1..c179d4e 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -144,6 +144,7 @@ func validateResourcesErrors(createdBlueprints []string, createdPages []string, func validateResourcesDoesNotExist(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { var errors []error if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil { + klog.Warningf("Integration with state key %s already exists", config.StateKey) return &AbortDefaultCreationError{Errors: []error{ fmt.Errorf("integration with state key %s already exists", config.StateKey), }} @@ -151,12 +152,14 @@ func validateResourcesDoesNotExist(portClient *cli.PortClient, defaults *Default for _, bp := range defaults.Blueprints { if _, err := blueprint.GetBlueprint(portClient, bp.Identifier); err == nil { + klog.Warningf("Blueprint with identifier %s already exists", bp.Identifier) errors = append(errors, fmt.Errorf("blueprint with identifier %s already exists", bp.Identifier)) } } for _, p := range defaults.Pages { if _, err := page.GetPage(portClient, p.Identifier); err == nil { + klog.Warningf("Page with identifier %s already exists", p.Identifier) errors = append(errors, fmt.Errorf("page with identifier %s already exists", p.Identifier)) } } @@ -169,6 +172,7 @@ func validateResourcesDoesNotExist(portClient *cli.PortClient, defaults *Default func createResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { if err := validateResourcesDoesNotExist(portClient, defaults, config); err != nil { + klog.Warningf("Failed to create resources: %v.", err.Errors) return err } @@ -189,8 +193,10 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por mutex.Lock() if err != nil { + klog.Warningf("Failed to create blueprint %s: %v", bp.Identifier, err.Error()) resourceErrors = append(resourceErrors, err) } else { + klog.Infof("Created blueprint %s", result.Identifier) createdBlueprints = append(createdBlueprints, result.Identifier) } mutex.Unlock() @@ -208,6 +214,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por go func(bp port.Blueprint) { defer waitGroup.Done() if _, err := blueprint.PatchBlueprint(portClient, bp); err != nil { + klog.Warningf("Failed to patch blueprint %s: %v", bp.Identifier, err.Error()) resourceErrors = append(resourceErrors, err) } }(bp) @@ -225,6 +232,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por go func(blueprintIdentifier string, scorecard port.Scorecard) { defer waitGroup.Done() if err := scorecards.CreateScorecard(portClient, blueprintIdentifier, scorecard); err != nil { + klog.Warningf("Failed to create scorecard for blueprint %s: %v", blueprintIdentifier, err.Error()) resourceErrors = append(resourceErrors, err) } }(blueprintScorecards.Blueprint, scorecard) @@ -241,8 +249,10 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por go func(p port.Page) { defer waitGroup.Done() if err := page.CreatePage(portClient, p); err != nil { + klog.Warningf("Failed to create page %s: %v", p.Identifier, err.Error()) resourceErrors = append(resourceErrors, err) } else { + klog.Infof("Created page %s", p.Identifier) createdPages = append(createdPages, p.Identifier) } }(pageToCreate) @@ -254,7 +264,7 @@ func createResources(portClient *cli.PortClient, defaults *Defaults, config *por } if err := integration.CreateIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { - klog.Infof("Failed to create resources: %v.", err.Error()) + klog.Warningf("Failed to create integration with default configuration. state key %s: %v", config.StateKey, err.Error()) return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, PagesToRollback: createdPages, Errors: []error{err}} } diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go index 7a7c07e..dc75d16 100644 --- a/pkg/defaults/defaults_test.go +++ b/pkg/defaults/defaults_test.go @@ -192,7 +192,7 @@ func Test_InitIntegration_ExistingIntegration(t *testing.T) { checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) } -func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { +func Test_InitIntegration_LocalResourcesConfiguration(t *testing.T) { f := NewFixture(t) err := integration.CreateIntegration(f.portClient, f.stateKey, "", nil) if err != nil { @@ -233,7 +233,7 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) } -func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { +func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { f := NewFixture(t) err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", nil) if err != nil { @@ -253,3 +253,49 @@ func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_E checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) } + +func Test_InitIntegration_LocalResourcesConfiguration_ExistingIntegration_WithConfiguration_WithOverwriteConfigurationOnRestartFlag(t *testing.T) { + f := NewFixture(t) + expectedConfig := &port.IntegrationAppConfig{ + Resources: []port.Resource{ + { + Kind: "workload", + Port: port.Port{ + Entity: port.EntityMappings{ + Mappings: []port.EntityMapping{ + { + Identifier: "\"workload\"", + Title: "\"Workload\"", + Blueprint: "\"workload\"", + Icon: "\"Microservice\"", + Properties: map[string]string{ + "namespace": "\"default\"", + }, + }, + }, + }, + }, + }, + }, + } + err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", expectedConfig) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + + expectedConfig.Resources[0].Kind = "namespace" + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "KAFKA", + Resources: expectedConfig.Resources, + CreateDefaultResources: true, + OverwriteConfigurationOnRestart: true, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + assert.Equal(t, expectedConfig.Resources, i.Config.Resources) + + checkResourcesDoesNotExist(f, []string{"workload", "namespace", "cluster"}, []string{"workload_overview_dashboard", "availability_scorecard_dashboard"}) +} diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go index e0f5521..f1b4d0c 100644 --- a/pkg/defaults/init.go +++ b/pkg/defaults/init.go @@ -54,7 +54,7 @@ func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) // Handle a deprecated case where resources are provided in config file and integration exists from previous //versions without a config - if existingIntegration.Config == nil { + if existingIntegration.Config == nil || applicationConfig.OverwriteConfigurationOnRestart { klog.Infof("Integration exists without a config, patching it with default config: %v", defaultIntegrationConfig) integrationPatch.Config = defaultIntegrationConfig } diff --git a/pkg/port/models.go b/pkg/port/models.go index f631be6..57d1966 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -182,7 +182,7 @@ type EntityMapping struct { Identifier string `json:"identifier"` Title string `json:"title"` Blueprint string `json:"blueprint"` - Icon string `json:"icon,omitempty"` + Icon string `json:"icon,omitempty"` Team string `json:"team,omitempty"` Properties map[string]string `json:"properties,omitempty"` Relations map[string]string `json:"relations,omitempty"` @@ -227,14 +227,13 @@ type IntegrationAppConfig struct { } type Config struct { - ResyncInterval uint - StateKey string - EventListenerType string - CreateDefaultResources bool - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - Resources []Resource `json:"resources,omitempty"` - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - DeleteDependents bool `json:"deleteDependents,omitempty"` - // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. - CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` + ResyncInterval uint `yaml:"resyncInterval,omitempty"` + StateKey string `yaml:"stateKey,omitempty"` + EventListenerType string `yaml:"eventListenerType,omitempty"` + CreateDefaultResources bool `yaml:"createDefaultResources,omitempty"` + OverwriteConfigurationOnRestart bool `yaml:"overwriteConfigurationOnRestart,omitempty"` + // These Configurations are used only for setting up the Integration on installation or when using OverwriteConfigurationOnRestart flag. + Resources []Resource `yaml:"resources,omitempty"` + DeleteDependents bool `yaml:"deleteDependents,omitempty"` + CreateMissingRelatedEntities bool `yaml:"createMissingRelatedEntities,omitempty"` }