diff --git a/.gitignore b/.gitignore index d51d5c5..790622b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ __debug_bin config.yaml deployments/k8s + +.env \ No newline at end of file diff --git a/assets/defaults/appConfig.yaml b/assets/defaults/appConfig.yaml new file mode 100644 index 0000000..bf61c0d --- /dev/null +++ b/assets/defaults/appConfig.yaml @@ -0,0 +1,106 @@ +createMissingRelatedEntities: true +resources: + - kind: v1/namespaces + port: + entity: + mappings: + - blueprint: '"namespace"' + identifier: .metadata.name + "-" + env.CLUSTER_NAME + properties: + creationTimestamp: .metadata.creationTimestamp + labels: .metadata.labels + relations: + Cluster: env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.name | startswith("kube") | not + - kind: v1/namespaces + port: + entity: + mappings: + - blueprint: '"cluster"' + identifier: env.CLUSTER_NAME + title: env.CLUSTER_NAME + selector: + query: .metadata.name | contains("kube-system") + - kind: apps/v1/deployments + port: + entity: + mappings: + - blueprint: '"workload"' + icon: '"Deployment"' + identifier: .metadata.name + "-Deployment-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + images: '(.spec.template.spec.containers | map({name, image, resources})) | map("\(.name): \(.image)")' + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"Deployment"' + labels: .metadata.labels + replicas: .spec.replicas + strategy: .spec.strategy.type + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not + - kind: apps/v1/daemonsets + port: + entity: + mappings: + - blueprint: '"workload"' + identifier: .metadata.name + "-DaemonSet-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"DaemonSet"' + labels: .metadata.labels + replicas: .spec.replicas + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not + - kind: apps/v1/statefulsets + port: + entity: + mappings: + - blueprint: '"workload"' + identifier: .metadata.name + "-StatefulSet-" + .metadata.namespace + "-" + env.CLUSTER_NAME + properties: + availableReplicas: .status.availableReplicas + containers: (.spec.template.spec.containers | map({name, image, resources})) + creationTimestamp: .metadata.creationTimestamp + hasLatest: .spec.template.spec.containers[].image | contains(":latest") + hasLimits: .spec.template.spec.containers | all(has("resources") and (.resources.limits.memory + and .resources.limits.cpu)) + hasPrivileged: .spec.template.spec.containers | [.[].securityContext.privileged] + | any + isHealthy: if .spec.replicas == .status.availableReplicas then "Healthy" + else "Unhealthy" end + kind: '"StatefulSet"' + labels: .metadata.labels + replicas: .spec.replicas + strategyConfig: .spec.strategy // {} + relations: + Namespace: .metadata.namespace + "-" + env.CLUSTER_NAME + title: .metadata.name + selector: + query: .metadata.namespace | startswith("kube") | not \ No newline at end of file diff --git a/assets/defaults/blueprints.json b/assets/defaults/blueprints.json new file mode 100644 index 0000000..3e7fca3 --- /dev/null +++ b/assets/defaults/blueprints.json @@ -0,0 +1,151 @@ +[ + { + "identifier": "cluster", + "description": "This blueprint represents a Kubernetes Cluster", + "title": "Cluster", + "icon": "Cluster", + "schema": { + "properties": {}, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": {} + }, + { + "identifier": "namespace", + "description": "This blueprint represents a k8s Namespace", + "title": "Namespace", + "icon": "Environment", + "schema": { + "properties": { + "creationTimestamp": { + "type": "string", + "title": "Created", + "format": "date-time", + "description": "When the Namespace was created" + }, + "labels": { + "type": "object", + "title": "Labels", + "description": "Labels of the Namespace" + } + }, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": { + "Cluster": { + "title": "Cluster", + "description": "The namespace's Kubernetes cluster", + "target": "cluster", + "required": false, + "many": false + } + } + }, + { + "identifier": "workload", + "description": "This blueprint represents a k8s Workload. This includes all k8s objects which can create pods (deployments[replicasets], daemonsets, statefulsets...)", + "title": "Workload", + "icon": "Deployment", + "schema": { + "properties": { + "availableReplicas": { + "type": "number", + "title": "Running Replicas", + "description": "Current running replica count" + }, + "containers": { + "type": "array", + "title": "Containers", + "default": [], + "description": "The containers for each pod instance of the Workload" + }, + "creationTimestamp": { + "type": "string", + "title": "Created", + "format": "date-time", + "description": "When the Workload was created" + }, + "labels": { + "type": "object", + "title": "Labels", + "description": "Labels of the Workload" + }, + "replicas": { + "type": "number", + "title": "Wanted Replicas", + "description": "Wanted replica count" + }, + "strategy": { + "type": "string", + "title": "Strategy", + "description": "Rollout Strategy" + }, + "hasPrivileged": { + "type": "boolean", + "title": "Has Privileged Container" + }, + "hasLatest": { + "type": "boolean", + "title": "Has 'latest' tag", + "description": "Has Container with 'latest' as image tag" + }, + "hasLimits": { + "type": "boolean", + "title": "All containers have limits" + }, + "isHealthy": { + "type": "string", + "enum": [ + "Healthy", + "Unhealthy" + ], + "enumColors": { + "Healthy": "green", + "Unhealthy": "red" + }, + "title": "Workload Health" + }, + "kind": { + "title": "Workload Kind", + "description": "The kind of Workload", + "type": "string", + "enum": [ + "StatefulSet", + "DaemonSet", + "Deployment", + "Rollout" + ] + }, + "strategyConfig": { + "type": "object", + "title": "Strategy Config", + "description": "The workloads rollout strategy" + } + }, + "required": [] + }, + "mirrorProperties": { + "Cluster": { + "title": "Cluster", + "path": "Namespace.Cluster.$title" + }, + "namespace": { + "title": "Namespace", + "path": "Namespace.$title" + } + }, + "calculationProperties": {}, + "relations": { + "Namespace": { + "title": "Namespace", + "target": "namespace", + "required": false, + "many": false + } + } + } +] \ No newline at end of file diff --git a/assets/defaults/scorecards.json b/assets/defaults/scorecards.json new file mode 100644 index 0000000..f5bc2da --- /dev/null +++ b/assets/defaults/scorecards.json @@ -0,0 +1,139 @@ +[ + { + "blueprint": "workload", + "data": [ + { + "identifier": "configuration", + "title": "Configuration Checks", + "rules": [ + { + "identifier": "notPrivileged", + "title": "No privilged containers", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasPrivileged", + "operator": "!=", + "value": true + } + ] + } + }, + { + "identifier": "hasLimits", + "title": "All containers have CPU and Memory limits", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasLimits", + "operator": "=", + "value": true + } + ] + } + }, + { + "identifier": "notDefaultNamespace", + "title": "Not in 'default' namespace", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "namespace", + "operator": "!=", + "value": "default" + } + ] + } + }, + { + "identifier": "rolloutStrategy", + "title": "Using Rolling update strategy", + "level": "Silver", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "strategy", + "operator": "=", + "value": "RollingUpdate" + } + ] + } + }, + { + "identifier": "imageTag", + "title": "Doesn't have a container with image tag 'latest'", + "level": "Gold", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "hasLatest", + "operator": "!=", + "value": "false" + } + ] + } + } + ] + }, + { + "identifier": "highAvailability", + "title": "High Availability", + "rules": [ + { + "identifier": "highAvalabilityB", + "title": "Highly Available", + "level": "Bronze", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 1 + } + ] + } + }, + { + "identifier": "highAvalabilityS", + "title": "Highly Available", + "level": "Silver", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 2 + } + ] + } + }, + { + "identifier": "highAvalabilityG", + "title": "Highly Available", + "level": "Gold", + "query": { + "combinator": "and", + "conditions": [ + { + "property": "replicas", + "operator": ">=", + "value": 3 + } + ] + } + } + ] + } + ] + } +] \ No newline at end of file diff --git a/go.mod b/go.mod index 2ce19e9..a23c46c 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/google/gofuzz v1.1.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/itchyny/timefmt-go v0.1.4 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/go.sum b/go.sum index 399958e..922bf79 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,8 @@ github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/main.go b/main.go index 7d19fab..8478f89 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,11 @@ package main import ( - "flag" + "errors" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/defaults" "github.com/port-labs/port-k8s-exporter/pkg/event_handler" - "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" - "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -16,39 +15,33 @@ import ( ) func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { - apiConfig, err := integration.GetIntegrationConfig(portClient, exporterConfig.StateKey) + i, err := integration.GetIntegration(portClient, exporterConfig.StateKey) if err != nil { - klog.Fatalf("Error getting K8s integration config: %s", err.Error()) + return nil, fmt.Errorf("error getting Port integration: %v", err) } + if i.Config == nil { + return nil, errors.New("integration config is nil") - cli.WithDeleteDependents(apiConfig.DeleteDependents)(portClient) - cli.WithCreateMissingRelatedEntities(apiConfig.CreateMissingRelatedEntities)(portClient) + } + + cli.WithDeleteDependents(i.Config.DeleteDependents)(portClient) + cli.WithCreateMissingRelatedEntities(i.Config.CreateMissingRelatedEntities)(portClient) - newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) + newHandler := handlers.NewControllersHandler(exporterConfig, i.Config, k8sClient, portClient) newHandler.Handle() return newHandler, nil } -func createEventListener(stateKey string, eventListenerType string, portClient *cli.PortClient) (event_handler.IListener, error) { - klog.Infof("Received event listener type: %s", eventListenerType) - switch eventListenerType { - case "KAFKA": - return consumer.NewEventListener(stateKey, portClient) - case "POLLING": - return polling.NewEventListener(stateKey, portClient), nil - default: - return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) - } - -} - func main() { klog.InitFlags(nil) k8sConfig := k8s.NewKubeConfig() + applicationConfig, err := config.NewConfiguration() + if err != nil { + klog.Fatalf("Error getting application config: %s", err.Error()) + } - exporterConfig, _ := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) clientConfig, err := k8sConfig.ClientConfig() if err != nil { klog.Fatalf("Error getting K8s client config: %s", err.Error()) @@ -61,32 +54,25 @@ func main() { portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret), - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey)), + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", applicationConfig.StateKey)), ) if err != nil { klog.Fatalf("Error building Port client: %s", err.Error()) } - _, err = integration.GetIntegrationConfig(portClient, exporterConfig.StateKey) - if err != nil { - if exporterConfig == nil { - klog.Fatalf("The integration does not exist and no config file was provided") - } - err = integration.NewIntegration(portClient, exporterConfig.StateKey, exporterConfig.EventListenerType, exporterConfig.Resources) - if err != nil { - klog.Fatalf("Error creating K8s integration: %s", err.Error()) - } + if err := defaults.InitIntegration(portClient, applicationConfig); err != nil { + klog.Fatalf("Error initializing Port integration: %s", err.Error()) } - eventListener, err := createEventListener(exporterConfig.StateKey, exporterConfig.EventListenerType, portClient) + eventListener, err := event_handler.CreateEventListener(applicationConfig.StateKey, applicationConfig.EventListenerType, portClient) if err != nil { klog.Fatalf("Error creating event listener: %s", err.Error()) } klog.Info("Starting controllers handler") err = event_handler.Start(eventListener, func() (event_handler.IStoppableRsync, error) { - return initiateHandler(exporterConfig, k8sClient, portClient) + return initiateHandler(applicationConfig, k8sClient, portClient) }) if err != nil { @@ -96,5 +82,4 @@ func main() { func init() { config.Init() - flag.Parse() } diff --git a/pkg/config/config.go b/pkg/config/config.go index fe8ea14..aa753be 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,11 +1,25 @@ package config +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/joho/godotenv" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "strings" +) + var KafkaConfig = &KafkaConfiguration{} var PollingListenerRate uint var ApplicationConfig = &ApplicationConfiguration{} func Init() { + _ = godotenv.Load() + + NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") + // Kafka listener Configuration NewString(&KafkaConfig.Brokers, "event-listener-brokers", "localhost:9092", "Kafka event listener brokers") NewString(&KafkaConfig.SecurityProtocol, "event-listener-security-protocol", "plaintext", "Kafka event listener security protocol") @@ -21,5 +35,41 @@ func Init() { NewString(&ApplicationConfig.PortBaseURL, "port-base-url", "https://api.getport.io", "Port base URL. Optional.") NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") - NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") + NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") + + // Deprecated + NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.") + NewBool(&ApplicationConfig.CreateMissingRelatedEntities, "create-missing-related-entities", false, "Create missing related entities. Optional.") + + flag.Parse() +} + +func NewConfiguration() (*port.Config, error) { + overrides := &port.Config{ + StateKey: ApplicationConfig.StateKey, + EventListenerType: ApplicationConfig.EventListenerType, + CreateDefaultResources: ApplicationConfig.CreateDefaultResources, + ResyncInterval: ApplicationConfig.ResyncInterval, + CreateMissingRelatedEntities: ApplicationConfig.CreateMissingRelatedEntities, + DeleteDependents: ApplicationConfig.DeleteDependents, + } + + c, err := GetConfigFile(ApplicationConfig.ConfigFilePath) + var fileNotFoundError *FileNotFoundError + if errors.As(err, &fileNotFoundError) { + return overrides, nil + } + v, err := json.Marshal(overrides) + if err != nil { + return nil, fmt.Errorf("failed loading configuration: %w", err) + } + + err = json.Unmarshal(v, &c) + if err != nil { + return nil, fmt.Errorf("failed loading configuration: %w", err) + } + + c.StateKey = strings.ToLower(c.StateKey) + + return c, nil } diff --git a/pkg/config/models.go b/pkg/config/models.go index f57eef1..4626e05 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -1,5 +1,7 @@ package config +import "github.com/port-labs/port-k8s-exporter/pkg/port" + type KafkaConfiguration struct { Brokers string SecurityProtocol string @@ -11,11 +13,18 @@ type KafkaConfiguration struct { } type ApplicationConfiguration struct { - ConfigFilePath string - StateKey string - ResyncInterval uint - PortBaseURL string - PortClientId string - PortClientSecret string - EventListenerType string + ConfigFilePath string + StateKey string + ResyncInterval uint + PortBaseURL string + PortClientId string + PortClientSecret string + EventListenerType string + CreateDefaultResources bool + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + Resources []port.Resource + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + DeleteDependents bool `json:"deleteDependents,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` } diff --git a/pkg/config/utils.go b/pkg/config/utils.go index ba61c39..bf1f163 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -5,7 +5,6 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/port" "gopkg.in/yaml.v2" - "k8s.io/klog/v2" "k8s.io/utils/strings/slices" "os" "strings" @@ -17,7 +16,7 @@ func prepareEnvKey(key string) string { newKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) if slices.Contains(keys, newKey) { - klog.Fatalf("Application Error : Found duplicate config key: %s", newKey) + panic("Application Error : Found duplicate config key: " + newKey) } keys = append(keys, newKey) @@ -34,6 +33,11 @@ func NewUInt(v *uint, key string, defaultValue uint, description string) { flag.UintVar(v, key, value, description) } +func NewBool(v *bool, key string, defaultValue bool, description string) { + value := goutils.GetBoolEnvOrDefault(prepareEnvKey(key), defaultValue) + flag.BoolVar(v, key, value, description) +} + type FileNotFoundError struct { s string } @@ -42,12 +46,8 @@ func (e *FileNotFoundError) Error() string { return e.s } -func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { - c := &port.Config{ - ResyncInterval: resyncInterval, - StateKey: stateKey, - EventListenerType: eventListenerType, - } +func GetConfigFile(filepath string) (*port.Config, error) { + c := &port.Config{} config, err := os.ReadFile(filepath) if err != nil { return c, &FileNotFoundError{err.Error()} diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go new file mode 100644 index 0000000..0e99eb8 --- /dev/null +++ b/pkg/defaults/defaults.go @@ -0,0 +1,238 @@ +package defaults + +import ( + "encoding/json" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/blueprint" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "github.com/port-labs/port-k8s-exporter/pkg/port/scorecards" + "gopkg.in/yaml.v3" + "k8s.io/klog/v2" + "os" + "sync" +) + +type ScorecardDefault struct { + Blueprint string `json:"blueprint"` + Scorecards []port.Scorecard `json:"data"` +} + +type Defaults struct { + Blueprints []port.Blueprint + Scorecards []ScorecardDefault + AppConfig *port.IntegrationAppConfig +} + +var BlueprintsAsset = "assets/defaults/blueprints.json" +var ScorecardsAsset = "assets/defaults/scorecards.json" +var AppConfigAsset = "assets/defaults/appConfig.yaml" + +func getDefaults() (*Defaults, error) { + var bp []port.Blueprint + file, err := os.ReadFile(BlueprintsAsset) + if err != nil { + klog.Infof("No default blueprints found. Skipping...") + } else { + err = json.Unmarshal(file, &bp) + if err != nil { + return nil, err + } + } + + var sc []ScorecardDefault + file, err = os.ReadFile(ScorecardsAsset) + if err != nil { + klog.Infof("No default scorecards found. Skipping...") + } else { + err = json.Unmarshal(file, &sc) + if err != nil { + return nil, err + } + } + + var appConfig *port.IntegrationAppConfig + file, err = os.ReadFile(AppConfigAsset) + if err != nil { + klog.Infof("No default appConfig found. Skipping...") + } else { + err = yaml.Unmarshal(file, &appConfig) + if err != nil { + return nil, err + } + } + + return &Defaults{ + Blueprints: bp, + Scorecards: sc, + AppConfig: appConfig, + }, nil +} + +// deconstructBlueprintsToCreationSteps takes a list of blueprints and returns a list of blueprints with only the +// required fields for creation, a list of blueprints with the required fields for creation and relations, and a list +// of blueprints with all fields for creation, relations, and calculation properties. +// This is done because there might be a case where a blueprint has a relation to another blueprint that +// hasn't been created yet. +func deconstructBlueprintsToCreationSteps(rawBlueprints []port.Blueprint) ([]port.Blueprint, [][]port.Blueprint) { + var ( + bareBlueprints []port.Blueprint + withRelations []port.Blueprint + fullBlueprints []port.Blueprint + ) + + for _, bp := range rawBlueprints { + bareBlueprint := port.Blueprint{ + Identifier: bp.Identifier, + Title: bp.Title, + Icon: bp.Icon, + Description: bp.Description, + Schema: bp.Schema, + } + bareBlueprints = append(bareBlueprints, bareBlueprint) + + withRelation := bareBlueprint + withRelation.Relations = bp.Relations + withRelations = append(withRelations, withRelation) + + fullBlueprint := withRelation + fullBlueprint.AggregationProperties = bp.AggregationProperties + fullBlueprint.CalculationProperties = bp.CalculationProperties + fullBlueprint.MirrorProperties = bp.MirrorProperties + fullBlueprints = append(fullBlueprints, fullBlueprint) + } + + return bareBlueprints, [][]port.Blueprint{withRelations, fullBlueprints} +} + +type AbortDefaultCreationError struct { + BlueprintsToRollback []string + Errors []error +} + +func (e *AbortDefaultCreationError) Error() string { + return "AbortDefaultCreationError" +} + +func validateBlueprintErrors(createdBlueprints []string, blueprintErrors []error) *AbortDefaultCreationError { + if len(blueprintErrors) > 0 { + for _, err := range blueprintErrors { + klog.Infof("Failed to create resources: %v.", err.Error()) + } + return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: blueprintErrors} + } + return nil +} + +func createResources(portClient *cli.PortClient, defaults *Defaults, config *port.Config) *AbortDefaultCreationError { + if _, err := integration.GetIntegration(portClient, config.StateKey); err == nil { + return &AbortDefaultCreationError{Errors: []error{ + fmt.Errorf("integration with state key %s already exists", config.StateKey), + }} + } + + bareBlueprints, patchStages := deconstructBlueprintsToCreationSteps(defaults.Blueprints) + + waitGroup := sync.WaitGroup{} + + var blueprintErrors []error + var createdBlueprints []string + mutex := sync.Mutex{} + + for _, bp := range bareBlueprints { + waitGroup.Add(1) + go func(bp port.Blueprint) { + defer waitGroup.Done() + result, err := blueprint.NewBlueprint(portClient, bp) + + mutex.Lock() + if err != nil { + blueprintErrors = append(blueprintErrors, err) + } else { + createdBlueprints = append(createdBlueprints, result.Identifier) + } + mutex.Unlock() + }(bp) + } + waitGroup.Wait() + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + + for _, patchStage := range patchStages { + for _, bp := range patchStage { + waitGroup.Add(1) + go func(bp port.Blueprint) { + defer waitGroup.Done() + if _, err := blueprint.PatchBlueprint(portClient, bp); err != nil { + blueprintErrors = append(blueprintErrors, err) + } + }(bp) + } + waitGroup.Wait() + } + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + + for _, blueprintScorecards := range defaults.Scorecards { + for _, scorecard := range blueprintScorecards.Scorecards { + waitGroup.Add(1) + go func(blueprintIdentifier string, scorecard port.Scorecard) { + defer waitGroup.Done() + if err := scorecards.CreateScorecard(portClient, blueprintIdentifier, scorecard); err != nil { + blueprintErrors = append(blueprintErrors, err) + } + }(blueprintScorecards.Blueprint, scorecard) + } + } + waitGroup.Wait() + + if err := validateBlueprintErrors(createdBlueprints, blueprintErrors); err != nil { + return err + } + + if err := integration.CreateIntegration(portClient, config.StateKey, config.EventListenerType, defaults.AppConfig); err != nil { + klog.Infof("Failed to create resources: %v.", err.Error()) + return &AbortDefaultCreationError{BlueprintsToRollback: createdBlueprints, Errors: []error{err}} + } + + return nil +} + +func initializeDefaults(portClient *cli.PortClient, config *port.Config) error { + defaults, err := getDefaults() + if err != nil { + return err + } + + if err := createResources(portClient, defaults, config); err != nil { + klog.Infof("Failed to create resources. Rolling back blueprints: %v", err.BlueprintsToRollback) + var rollbackWg sync.WaitGroup + for _, identifier := range err.BlueprintsToRollback { + rollbackWg.Add(1) + go func(identifier string) { + defer rollbackWg.Done() + if err := blueprint.DeleteBlueprint(portClient, identifier); err != nil { + klog.Warningf("Failed to rollback blueprint %s creation: %v", identifier, err) + } + }(identifier) + } + rollbackWg.Wait() + return &ExceptionGroup{Message: err.Error(), Errors: err.Errors} + } + + return nil +} + +type ExceptionGroup struct { + Message string + Errors []error +} + +func (e *ExceptionGroup) Error() string { + return e.Message +} diff --git a/pkg/defaults/defaults_test.go b/pkg/defaults/defaults_test.go new file mode 100644 index 0000000..c220e7c --- /dev/null +++ b/pkg/defaults/defaults_test.go @@ -0,0 +1,190 @@ +package defaults + +import ( + "fmt" + guuid "github.com/google/uuid" + "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/blueprint" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + _ "github.com/port-labs/port-k8s-exporter/test_utils" + "github.com/stretchr/testify/assert" + "testing" +) + +type Fixture struct { + t *testing.T + portClient *cli.PortClient + stateKey string +} + +func checkBlueprintsDoesNotExist(f *Fixture, blueprints []string) { + for _, bp := range blueprints { + _, err := blueprint.GetBlueprint(f.portClient, bp) + if err != nil { + _ = blueprint.DeleteBlueprint(f.portClient, bp) + } + assert.NotNil(f.t, err) + } +} + +func NewFixture(t *testing.T) *Fixture { + stateKey := guuid.NewString() + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) + if err != nil { + t.Errorf("Error building Port client: %s", err.Error()) + } + + deleteDefaultResources(portClient, stateKey) + return &Fixture{ + t: t, + portClient: portClient, + stateKey: stateKey, + } +} + +func (f *Fixture) CreateIntegration() { + err := integration.CreateIntegration(f.portClient, f.stateKey, "", &port.IntegrationAppConfig{ + Resources: []port.Resource{}, + }) + + if err != nil { + f.t.Errorf("Error creating Port integration: %s", err.Error()) + } +} + +func (f *Fixture) CleanIntegration() { + _ = integration.DeleteIntegration(f.portClient, f.stateKey) +} + +func deleteDefaultResources(portClient *cli.PortClient, stateKey string) { + _ = integration.DeleteIntegration(portClient, stateKey) + _ = blueprint.DeleteBlueprint(portClient, "workload") + _ = blueprint.DeleteBlueprint(portClient, "namespace") + _ = blueprint.DeleteBlueprint(portClient, "cluster") +} + +func Test_InitIntegration_InitDefaults(t *testing.T) { + f := NewFixture(t) + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: true, + }) + assert.Nil(t, e) + + _, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "workload") + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "namespace") + assert.Nil(t, err) + + _, err = blueprint.GetBlueprint(f.portClient, "cluster") + assert.Nil(t, err) +} + +func Test_InitIntegration_InitDefaults_CreateDefaultResources_False(t *testing.T) { + f := NewFixture(t) + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: false, + }) + assert.Nil(t, e) + + _, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) +} + +func Test_InitIntegration_FailingInitDefaults(t *testing.T) { + f := NewFixture(t) + if _, err := blueprint.NewBlueprint(f.portClient, port.Blueprint{ + Identifier: "workload", + Title: "Workload", + Schema: port.BlueprintSchema{ + Properties: map[string]port.BlueprintProperty{}, + }, + }); err != nil { + t.Errorf("Error creating Port blueprint: %s", err.Error()) + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + CreateDefaultResources: true, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.True(t, nil == i.Config.Resources) + assert.Nil(t, err) + + checkBlueprintsDoesNotExist(f, []string{"namespace", "cluster"}) +} + +func Test_InitIntegration_DeprecatedResourcesConfiguration(t *testing.T) { + f := NewFixture(t) + err := integration.CreateIntegration(f.portClient, f.stateKey, "", nil) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + expectedResources := []port.Resource{ + { + Kind: "workload", + Port: port.Port{ + Entity: port.EntityMappings{ + Mappings: []port.EntityMapping{ + { + Identifier: "workload", + Title: "Workload", + Blueprint: "workload", + Properties: map[string]string{ + "namespace": "default", + }, + }, + }, + }, + }, + }, + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "POLLING", + Resources: expectedResources, + CreateDefaultResources: true, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Equal(t, expectedResources, i.Config.Resources) + assert.Nil(t, err) + + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) +} + +func Test_InitIntegration_DeprecatedResourcesConfiguration_ExistingIntegration_EmptyConfiguration(t *testing.T) { + f := NewFixture(t) + err := integration.CreateIntegration(f.portClient, f.stateKey, "POLLING", nil) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + e := InitIntegration(f.portClient, &port.Config{ + StateKey: f.stateKey, + EventListenerType: "KAFKA", + Resources: nil, + CreateDefaultResources: true, + }) + assert.Nil(t, e) + + i, err := integration.GetIntegration(f.portClient, f.stateKey) + assert.Nil(t, err) + assert.Equal(t, "KAFKA", i.EventListener.Type) + + checkBlueprintsDoesNotExist(f, []string{"workload", "namespace", "cluster"}) +} diff --git a/pkg/defaults/init.go b/pkg/defaults/init.go new file mode 100644 index 0000000..2a04953 --- /dev/null +++ b/pkg/defaults/init.go @@ -0,0 +1,55 @@ +package defaults + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "k8s.io/klog/v2" +) + +func getEventListenerConfig(eventListenerType string) *port.EventListenerSettings { + if eventListenerType == "KAFKA" { + return &port.EventListenerSettings{ + Type: eventListenerType, + } + } + return nil +} + +func InitIntegration(portClient *cli.PortClient, applicationConfig *port.Config) error { + existingIntegration, err := integration.GetIntegration(portClient, applicationConfig.StateKey) + defaultIntegrationConfig := &port.IntegrationAppConfig{ + Resources: applicationConfig.Resources, + DeleteDependents: applicationConfig.DeleteDependents, + CreateMissingRelatedEntities: applicationConfig.CreateMissingRelatedEntities, + } + + if err != nil { + // The exporter supports a deprecated case where resources are provided in config file and integration does not + // exist. If this is not the case, we support the new way of creating the integration with the default resources. + // Only one of the two cases can be true. + if defaultIntegrationConfig.Resources == nil && applicationConfig.CreateDefaultResources { + if err := initializeDefaults(portClient, applicationConfig); err != nil { + klog.Warningf("Error initializing defaults: %s", err.Error()) + klog.Warningf("The integration will start without default integration mapping and other default resources. Please create them manually in Port. ") + } else { + return nil + } + } + + // Handle a deprecated case where resources are provided in config file + return integration.CreateIntegration(portClient, applicationConfig.StateKey, applicationConfig.EventListenerType, defaultIntegrationConfig) + } else { + integrationPatch := &port.Integration{ + EventListener: getEventListenerConfig(applicationConfig.EventListenerType), + } + + // Handle a deprecated case where resources are provided in config file and integration exists from previous + //versions without a config + if existingIntegration.Config == nil { + integrationPatch.Config = defaultIntegrationConfig + } + + return integration.PatchIntegration(portClient, applicationConfig.StateKey, integrationPatch) + } +} diff --git a/pkg/event_handler/event_listener_factory.go b/pkg/event_handler/event_listener_factory.go new file mode 100644 index 0000000..e84fdb1 --- /dev/null +++ b/pkg/event_handler/event_listener_factory.go @@ -0,0 +1,21 @@ +package event_handler + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "k8s.io/klog/v2" +) + +func CreateEventListener(stateKey string, eventListenerType string, portClient *cli.PortClient) (IListener, error) { + klog.Infof("Received event listener type: %s", eventListenerType) + switch eventListenerType { + case "KAFKA": + return consumer.NewEventListener(stateKey, portClient) + case "POLLING": + return polling.NewEventListener(stateKey, portClient), nil + default: + return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) + } +} diff --git a/pkg/event_handler/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go index 759e84d..468bfd8 100644 --- a/pkg/event_handler/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -1,8 +1,9 @@ package polling import ( - "flag" "fmt" + _ "github.com/port-labs/port-k8s-exporter/test_utils" + guuid "github.com/google/uuid" "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -29,17 +30,17 @@ func (m *MockTicker) GetC() <-chan time.Time { } func NewFixture(t *testing.T, c chan time.Time) *Fixture { - config.Init() - flag.Parse() stateKey := guuid.NewString() - portClient, err := cli.New("https://api.getport.io", cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) if err != nil { t.Errorf("Error building Port client: %s", err.Error()) } _ = integration.DeleteIntegration(portClient, stateKey) - err = integration.NewIntegration(portClient, stateKey, "", []port.Resource{}) + err = integration.CreateIntegration(portClient, stateKey, "", &port.IntegrationAppConfig{ + Resources: []port.Resource{}, + }) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) } @@ -69,8 +70,10 @@ func TestPolling_DifferentConfiguration(t *testing.T) { time.Sleep(time.Millisecond * 500) assert.False(t, called) - _ = integration.UpdateIntegrationConfig(fixture.portClient, fixture.stateKey, &port.AppConfig{ - Resources: []port.Resource{}, + _ = integration.PatchIntegration(fixture.portClient, fixture.stateKey, &port.Integration{ + Config: &port.IntegrationAppConfig{ + Resources: []port.Resource{}, + }, }) c <- time.Now() diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index 3526492..be6c0fd 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -26,3 +26,16 @@ func GetUintEnvOrDefault(key string, defaultValue uint64) uint64 { } return result } + +func GetBoolEnvOrDefault(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.ParseBool(value) + if err != nil { + fmt.Printf("Using default value "+strconv.FormatBool(defaultValue)+" for "+key+". error parsing env variable %s: %s", key, err.Error()) + return defaultValue + } + return result +} diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index 385aff3..4c14395 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -21,7 +21,7 @@ type ControllersHandler struct { stopCh chan struct{} } -func NewControllersHandler(exporterConfig *port.Config, portConfig *port.AppConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { +func NewControllersHandler(exporterConfig *port.Config, portConfig *port.IntegrationAppConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { resync := time.Minute * time.Duration(exporterConfig.ResyncInterval) informersFactory := dynamicinformer.NewDynamicSharedInformerFactory(k8sClient.DynamicClient, resync) @@ -50,10 +50,6 @@ func NewControllersHandler(exporterConfig *port.Config, portConfig *port.AppConf controllers = append(controllers, controller) } - if len(controllers) == 0 { - klog.Fatalf("Failed to initiate a controller for all resources, exiting...") - } - controllersHandler := &ControllersHandler{ controllers: controllers, informersFactory: informersFactory, diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index f6dac62..32ec26e 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -1,13 +1,14 @@ package k8s import ( + "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + _ "github.com/port-labs/port-k8s-exporter/test_utils" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" k8sfake "k8s.io/client-go/dynamic/fake" - "os" "reflect" "strings" "testing" @@ -33,12 +34,12 @@ func newFixture(t *testing.T, portClientId string, portClientSecret string, reso kubeclient := k8sfake.NewSimpleDynamicClient(runtime.NewScheme()) if portClientId == "" { - portClientId = os.Getenv("PORT_CLIENT_ID") + portClientId = config.ApplicationConfig.PortClientId } if portClientSecret == "" { - portClientSecret = os.Getenv("PORT_CLIENT_SECRET") + portClientSecret = config.ApplicationConfig.PortClientSecret } - portClient, err := cli.New("https://api.getport.io", cli.WithHeader("User-Agent", "port-k8s-exporter/0.1"), + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithHeader("User-Agent", "port-k8s-exporter/0.1"), cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret)) if err != nil { t.Errorf("Error building Port client: %s", err.Error()) diff --git a/pkg/port/blueprint/blueprint.go b/pkg/port/blueprint/blueprint.go new file mode 100644 index 0000000..6bb44ad --- /dev/null +++ b/pkg/port/blueprint/blueprint.go @@ -0,0 +1,60 @@ +package blueprint + +import ( + "context" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" +) + +func NewBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + + bp, err := cli.CreateBlueprint(portClient, blueprint) + if err != nil { + return nil, fmt.Errorf("error creating Port blueprint: %v", err) + } + return bp, nil +} + +func PatchBlueprint(portClient *cli.PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + + bp, err := cli.PatchBlueprint(portClient, blueprint) + if err != nil { + return nil, fmt.Errorf("error patching Port blueprint: %v", err) + } + return bp, nil +} + +func DeleteBlueprint(portClient *cli.PortClient, blueprintIdentifier string) error { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return fmt.Errorf("error authenticating with Port: %v", err) + } + + err = cli.DeleteBlueprint(portClient, blueprintIdentifier) + if err != nil { + return fmt.Errorf("error deleting Port blueprint: %v", err) + } + return nil +} + +func GetBlueprint(portClient *cli.PortClient, blueprintIdentifier string) (*port.Blueprint, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + + bp, err := cli.GetBlueprint(portClient, blueprintIdentifier) + if err != nil { + return nil, fmt.Errorf("error getting Port blueprint: %v", err) + } + return bp, nil +} diff --git a/pkg/port/cli/blueprint.go b/pkg/port/cli/blueprint.go new file mode 100644 index 0000000..571ae2d --- /dev/null +++ b/pkg/port/cli/blueprint.go @@ -0,0 +1,64 @@ +package cli + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func CreateBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Post("v1/blueprints") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func PatchBlueprint(portClient *PortClient, blueprint port.Blueprint) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + SetBody(blueprint). + Patch(fmt.Sprintf("v1/blueprints/%s", blueprint.Identifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to patch blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} + +func DeleteBlueprint(portClient *PortClient, blueprintIdentifier string) error { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Delete(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return err + } + if !pb.OK { + return fmt.Errorf("failed to delete blueprint, got: %s", resp.Body()) + } + return nil +} + +func GetBlueprint(portClient *PortClient, blueprintIdentifier string) (*port.Blueprint, error) { + pb := &port.ResponseBody{} + resp, err := portClient.Client.R(). + SetResult(&pb). + Get(fmt.Sprintf("v1/blueprints/%s", blueprintIdentifier)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to get blueprint, got: %s", resp.Body()) + } + return &pb.Blueprint, nil +} diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index df38d9a..e158f6b 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -14,7 +14,7 @@ func parseIntegration(i *port.Integration) *port.Integration { } if i.EventListener.Type == "KAFKA" { - x.EventListener = port.EventListenerSettings{ + x.EventListener = &port.EventListenerSettings{ Type: i.EventListener.Type, } } @@ -52,20 +52,6 @@ func (c *PortClient) GetIntegration(stateKey string) (*port.Integration, error) return &pb.Integration, nil } -func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.AppConfig, error) { - pb := &port.ResponseBody{} - resp, err := c.Client.R(). - SetResult(&pb). - Get(fmt.Sprintf("v1/integration/%s", stateKey)) - if err != nil { - return nil, err - } - if !pb.OK { - return nil, fmt.Errorf("failed to get integration config, got: %s", resp.Body()) - } - return pb.Integration.Config, nil -} - func (c *PortClient) DeleteIntegration(stateKey string) error { resp, err := c.Client.R(). Delete(fmt.Sprintf("v1/integration/%s", stateKey)) @@ -78,17 +64,12 @@ func (c *PortClient) DeleteIntegration(stateKey string) error { return nil } -func (c *PortClient) UpdateConfig(stateKey string, config *port.AppConfig) error { - type Config struct { - Config *port.AppConfig `json:"config"` - } +func (c *PortClient) PatchIntegration(stateKey string, integration *port.Integration) error { pb := &port.ResponseBody{} resp, err := c.Client.R(). - SetBody(&Config{ - Config: config, - }). + SetBody(integration). SetResult(&pb). - Patch(fmt.Sprintf("v1/integration/%s/config", stateKey)) + Patch(fmt.Sprintf("v1/integration/%s", stateKey)) if err != nil { return err } diff --git a/pkg/port/cli/scorecards.go b/pkg/port/cli/scorecards.go new file mode 100644 index 0000000..f0fbee8 --- /dev/null +++ b/pkg/port/cli/scorecards.go @@ -0,0 +1,22 @@ +package cli + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func (c *PortClient) CreateScorecard(blueprintIdentifier string, scorecard port.Scorecard) (*port.Scorecard, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + SetBody(scorecard). + SetPathParam("blueprint", blueprintIdentifier). + Post("v1/blueprints/{blueprint}/scorecards") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create scorecard, got: %s", resp.Body()) + } + return &pb.Scorecard, nil +} diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 4a5c227..5af4ba1 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -7,17 +7,15 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, resources []port.Resource) error { +func CreateIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, appConfig *port.IntegrationAppConfig) error { integration := &port.Integration{ Title: stateKey, InstallationAppType: "K8S EXPORTER", InstallationId: stateKey, - EventListener: port.EventListenerSettings{ + EventListener: &port.EventListenerSettings{ Type: eventListenerType, }, - Config: &port.AppConfig{ - Resources: resources, - }, + Config: appConfig, } _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { @@ -31,21 +29,6 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy return nil } -// ToDo: remove this function -func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.AppConfig, error) { - _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) - if err != nil { - return nil, fmt.Errorf("error authenticating with Port: %v", err) - } - - apiConfig, err := portClient.GetIntegrationConfig(stateKey) - if err != nil { - return nil, fmt.Errorf("error getting Port integration config: %v", err) - } - - return apiConfig, nil -} - func GetIntegration(portClient *cli.PortClient, stateKey string) (*port.Integration, error) { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { @@ -73,15 +56,15 @@ func DeleteIntegration(portClient *cli.PortClient, stateKey string) error { return nil } -func UpdateIntegrationConfig(portClient *cli.PortClient, stateKey string, config *port.AppConfig) error { +func PatchIntegration(portClient *cli.PortClient, stateKey string, integration *port.Integration) error { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { return fmt.Errorf("error authenticating with Port: %v", err) } - err = portClient.UpdateConfig(stateKey, config) + err = portClient.PatchIntegration(stateKey, integration) if err != nil { - return fmt.Errorf("error updating Port integration config: %v", err) + return fmt.Errorf("error updating Port integration: %v", err) } return nil } diff --git a/pkg/port/models.go b/pkg/port/models.go index 2f06df7..fa2c952 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -28,13 +28,13 @@ type ( } Integration struct { - InstallationId string `json:"installationId,omitempty"` - Title string `json:"title,omitempty"` - Version string `json:"version,omitempty"` - InstallationAppType string `json:"installationAppType,omitempty"` - EventListener EventListenerSettings `json:"changelogDestination,omitempty"` - Config *AppConfig `json:"config,omitempty"` - UpdatedAt *time.Time `json:"updatedAt,omitempty"` + InstallationId string `json:"installationId,omitempty"` + Title string `json:"title,omitempty"` + Version string `json:"version,omitempty"` + InstallationAppType string `json:"installationAppType,omitempty"` + EventListener *EventListenerSettings `json:"changelogDestination,omitempty"` + Config *IntegrationAppConfig `json:"config,omitempty"` + UpdatedAt *time.Time `json:"updatedAt,omitempty"` } BlueprintProperty struct { @@ -57,10 +57,24 @@ type ( Path string `json:"path,omitempty"` } - BlueprintFormulaProperty struct { - Identifier string `json:"identifier,omitempty"` - Title string `json:"title,omitempty"` - Formula string `json:"formula,omitempty"` + BlueprintCalculationProperty struct { + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Calculation string `json:"calculation,omitempty"` + Colors map[string]string `json:"colors,omitempty"` + Colorized bool `json:"colorized,omitempty"` + Format string `json:"format,omitempty"` + Type string `json:"type,omitempty"` + } + + BlueprintAggregationProperty struct { + Title string `json:"title"` + Target string `json:"target"` + CalculationSpec interface{} `json:"calculationSpec"` + Query interface{} `json:"query,omitempty"` + Description string `json:"description,omitempty"` + Icon string `json:"icon,omitempty"` + Type string `json:"type,omitempty"` } BlueprintSchema struct { @@ -82,15 +96,16 @@ type ( Blueprint struct { Meta - Identifier string `json:"identifier,omitempty"` - Title string `json:"title"` - Icon string `json:"icon"` - Description string `json:"description"` - Schema BlueprintSchema `json:"schema"` - FormulaProperties map[string]BlueprintFormulaProperty `json:"formulaProperties"` - MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` - ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` - Relations map[string]Relation `json:"relations,omitempty"` + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Icon string `json:"icon"` + Description string `json:"description"` + Schema BlueprintSchema `json:"schema"` + CalculationProperties map[string]BlueprintCalculationProperty `json:"calculationProperties,omitempty"` + AggregationProperties map[string]BlueprintAggregationProperty `json:"aggregationProperties,omitempty"` + MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` + ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` + Relations map[string]Relation `json:"relations,omitempty"` } Action struct { @@ -104,6 +119,13 @@ type ( InvocationMethod *InvocationMethod `json:"invocationMethod,omitempty"` } + Scorecard struct { + Identifier string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Filter interface{} `json:"filter,omitempty"` + Rules []interface{} `json:"rules,omitempty"` + } + Relation struct { Identifier string `json:"identifier,omitempty"` Title string `json:"title,omitempty"` @@ -142,6 +164,7 @@ type ResponseBody struct { Integration Integration `json:"integration"` KafkaCredentials OrgKafkaCredentials `json:"credentials"` OrgDetails OrgDetails `json:"organization"` + Scorecard Scorecard `json:"scorecard"` } type EntityMapping struct { @@ -185,16 +208,21 @@ type AggregatedResource struct { KindConfigs []KindConfig } -type AppConfig struct { +type IntegrationAppConfig struct { DeleteDependents bool `json:"deleteDependents,omitempty"` CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` - Resources []Resource `json:"resources"` + Resources []Resource `json:"resources,omitempty"` } type Config struct { - ResyncInterval uint - StateKey string - EventListenerType string - // Deprecated: use AppConfig instead. Used for updating the Port integration config on startup. - Resources []Resource + ResyncInterval uint + StateKey string + EventListenerType string + CreateDefaultResources bool + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + Resources []Resource `json:"resources,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + DeleteDependents bool `json:"deleteDependents,omitempty"` + // Deprecated: use IntegrationAppConfig instead. Used for updating the Port integration config on startup. + CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` } diff --git a/pkg/port/scorecards/scorecards.go b/pkg/port/scorecards/scorecards.go new file mode 100644 index 0000000..ddd12b6 --- /dev/null +++ b/pkg/port/scorecards/scorecards.go @@ -0,0 +1,22 @@ +package scorecards + +import ( + "context" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" +) + +func CreateScorecard(portClient *cli.PortClient, blueprintIdentifier string, scorecard port.Scorecard) error { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return fmt.Errorf("error authenticating with Port: %v", err) + } + + _, err = portClient.CreateScorecard(blueprintIdentifier, scorecard) + if err != nil { + return fmt.Errorf("error creating Port integration: %v", err) + } + + return nil +} diff --git a/pkg/signal/signal.go b/pkg/signal/signal.go index 7b6ba80..ff2c225 100644 --- a/pkg/signal/signal.go +++ b/pkg/signal/signal.go @@ -4,31 +4,38 @@ import ( "fmt" "os" "os/signal" + "sync" "syscall" ) func SetupSignalHandler() (stopCh chan struct{}) { - + mutex := sync.Mutex{} stop := make(chan struct{}) gracefulStop := false shutdownCh := make(chan os.Signal, 2) signal.Notify(shutdownCh, os.Interrupt, syscall.SIGTERM) go func() { <-shutdownCh + mutex.Lock() if gracefulStop == false { fmt.Fprint(os.Stderr, "Received SIGTERM, exiting gracefully...\n") close(stop) } + mutex.Unlock() <-shutdownCh + mutex.Lock() if gracefulStop == false { fmt.Fprint(os.Stderr, "Received SIGTERM again, exiting forcefully...\n") os.Exit(1) } + mutex.Unlock() }() go func() { <-stop + mutex.Lock() gracefulStop = true + mutex.Unlock() close(shutdownCh) }() diff --git a/test_utils/testing_init.go b/test_utils/testing_init.go new file mode 100644 index 0000000..27873e0 --- /dev/null +++ b/test_utils/testing_init.go @@ -0,0 +1,20 @@ +package testing_init + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/config" + "os" + "path" + "runtime" + "testing" +) + +func init() { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..") + err := os.Chdir(dir) + if err != nil { + panic(err) + } + testing.Init() + config.Init() +}