From 09e208c6cfbde5b382fdc7bd8e1d500c7b727585 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Sun, 18 Aug 2024 18:04:47 -0500 Subject: [PATCH] ./hack/update-expected.sh --- ...odes-default.minimal.example.com_user_data | 2 +- ...-machinetype.minimal.example.com_user_data | 2 +- ...s_s3_object_cluster-completed.spec_content | 2 +- ...nimal.example.com-addons-bootstrap_content | 4 +- ...e.com-addons-karpenter.sh-k8s-1.19_content | 4345 +++++++++++++---- ...deupconfig-karpenter-nodes-default_content | 1 + ...karpenter-nodes-single-machinetype_content | 1 + .../update_cluster/karpenter/kubernetes.tf | 23 +- 8 files changed, 3379 insertions(+), 1001 deletions(-) diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data index 18218be647657..7bc83a92f6577 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data @@ -153,7 +153,7 @@ ConfigServer: - https://kops-controller.internal.minimal.example.com:3988/ InstanceGroupName: karpenter-nodes-default InstanceGroupRole: Node -NodeupConfigHash: B5rzCKtE6ohaJMn5dHwp4M7fyRbxz6OXfR9LzdQ/e5c= +NodeupConfigHash: JVAAsN3tX9eYDcFEzz8jRw325wWlYeqt//yHdS73qk4= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data index 08b885e83c7b3..cb05e98d98eeb 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data @@ -153,7 +153,7 @@ ConfigServer: - https://kops-controller.internal.minimal.example.com:3988/ InstanceGroupName: karpenter-nodes-single-machinetype InstanceGroupRole: Node -NodeupConfigHash: tycZwbphxXfkC5C+Ysp9n6kzd1QNHlj3nq2zTckmz0c= +NodeupConfigHash: ENv8wgAF56Ga7fH/Pjix6bgcqqhv8Xo+plwQWGQEcCM= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content index 1e138e6814d2f..40952eb9d8b7d 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content @@ -57,7 +57,7 @@ spec: karpenter: cpuRequest: 100m enabled: true - image: public.ecr.aws/karpenter/controller:v0.31.3 + image: public.ecr.aws/karpenter/controller:1.0.0 logEncoding: console logLevel: debug memoryLimit: 2Gi diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 645a618d3c199..ee499ef1aa18c 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -120,14 +120,12 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: 09f06376ef7bfcb706ec648daadae641bf9650d7ab10b6b58d7cd33c0c224867 + manifestHash: b73e89841f52e80393b7bcb1018baf4a873bba432f404d8a52da098e9d455486 name: karpenter.sh prune: kinds: - kind: ConfigMap labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops - namespaces: - - kube-system - kind: Service labelSelector: addon.kops.k8s.io/name=karpenter.sh,app.kubernetes.io/managed-by=kops namespaces: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index e6fbe2881b7e4..b017a6c4e9980 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -2,447 +2,924 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.15.0 creationTimestamp: null labels: addon.kops.k8s.io/name: karpenter.sh app.kubernetes.io/managed-by: kops k8s-addon: karpenter.sh - name: provisioners.karpenter.sh + name: ec2nodeclasses.karpenter.k8s.aws spec: - group: karpenter.sh + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 + conversionReviewVersions: + - v1beta1 + - v1 + group: karpenter.k8s.aws names: categories: - karpenter - kind: Provisioner - listKind: ProvisionerList - plural: provisioners - singular: provisioner + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .spec.providerRef.name - name: Template + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready type: string - - jsonPath: .spec.weight - name: Weight + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role priority: 1 type: string - name: v1alpha5 + name: v1 schema: openAPIV3Schema: - description: Provisioner is the Schema for the Provisioners API + description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: ProvisionerSpec is the top level provisioner specification. - Provisioners launch nodes in response to pods that are unschedulable. - A single provisioner is capable of managing a diverse set of nodes. - Node properties are determined from a combination of provisioner and - pod scheduling constraints. + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. properties: - annotations: - additionalProperties: - type: string - description: Annotations are applied to every node. - type: object - consolidation: - description: Consolidation are the consolidation parameters - properties: - enabled: - description: Enabled enables consolidation if it has been set - type: boolean - type: object - kubeletConfiguration: - description: KubeletConfiguration are options passed to the kubelet - when provisioning nodes + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the + format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: + ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', + ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] + ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', + 'alias'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a + combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a + combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set + up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS + volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: KMSKeyID (ARN) of the symmetric Key Management + Service (KMS) CMK used for encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is + enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. items: type: string type: array - containerRuntime: - description: ContainerRuntime is the container runtime to be used - with your worker nodes. - type: string cpuCFSQuota: description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. type: boolean evictionHard: additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ type: string description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. format: int32 type: integer evictionSoft: additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ type: string description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionSoftGracePeriod: additionalProperties: type: string description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. format: int32 maximum: 100 minimum: 0 type: integer imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent format: int32 maximum: 100 minimum: 0 type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string description: KubeReserved contains resources reserved for Kubernetes system components. type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. format: int32 minimum: 0 type: integer podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. format: int32 minimum: 0 type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - labels: - additionalProperties: - type: string - description: Labels are layered with Requirements and applied to every - node. - type: object - limits: - description: Limits define a set of bounds for provisioning capacity. - properties: - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources contains all the allocatable resources - that Karpenter supports for limiting. - type: object - type: object - provider: - description: Provider contains fields specific to your cloudprovider. - type: object - x-kubernetes-preserve-unknown-fields: true - providerRef: - description: ProviderRef is a reference to a dedicated CRD for the - chosen provider, that holds additional configuration options + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: - apiVersion: - description: API version of the referent + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional type: string - required: - - name type: object - requirements: - description: Requirements are layered with Labels and applied to every - node. + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group + selector terms. The terms are ORed. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - key: - description: The label key that the selector applies to. + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ type: string - operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. type: string - values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. - items: + tags: + additionalProperties: type: string - type: array - required: - - key - - operator + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. + The terms are ORed. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ type: string - required: - - effect - - key + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array - taints: - description: Taints will be applied to every node launched by the - Provisioner. If specified, the provisioner will not provision nodes - for pods that do not have matching tolerations. Additional taints - will be created that match pod tolerations on a per-node basis. + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and + launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) + && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. + You must delete and recreate this node class if you want to change + this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) + && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 + alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' + || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an + AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' + || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using + a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using + a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using + a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) + && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily + == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain + an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: AMI contains resolved AMI selector values utilized + for node launch properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time + id: + description: ID of the AMI type: string - value: - description: The taint value corresponding to the taint key. + name: + description: Name of the AMI type: string + requirements: + description: Requirements of the AMI to be utilized on an instance + type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array required: - - effect - - key + - id + - requirements type: object type: array - ttlSecondsAfterEmpty: - description: "TTLSecondsAfterEmpty is the number of seconds the controller - will wait before attempting to delete a node, measured from when - the node is detected to be empty. A Node is considered to be empty - when it does not have pods scheduled to it, excluding daemonsets. - \n Termination due to no utilization is disabled if this field is - not set." - format: int64 - type: integer - ttlSecondsUntilExpired: - description: "TTLSecondsUntilExpired is the number of seconds the - controller will wait before terminating a node, measured from when - the node is created. This is useful to implement features like eventually - consistent node upgrade, memory leak protection, and disruption - testing. \n Termination due to expiration is disabled if this field - is not set." - format: int64 - type: integer - weight: - description: Weight is the priority given to the provisioner during - scheduling. A higher numerical weight indicates that this provisioner - will be ordered ahead of other provisioners with lower weights. - A provisioner with no weight will be treated as if it is a provisioner - with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - type: object - status: - description: ProvisionerStatus defines the observed state of Provisioner - properties: conditions: - description: Conditions is the set of conditions required for this - provisioner to scale its target, and indicates whether or not those - conditions are met. + description: Conditions contains signals for health and readiness items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + description: Condition aliases the upstream type and adds additional + helper methods properties: lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime + - message + - reason - status - type type: object type: array - lastScaleTime: - description: LastScaleTime is the last time the Provisioner scaled - the number of nodes - format: date-time + instanceProfile: + description: InstanceProfile contains the resolved instance profile + for the role type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object + securityGroups: + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector + values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized + for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array type: object type: object served: true storage: true subresources: status: {} - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: awsnodetemplates.karpenter.k8s.aws -spec: - group: karpenter.k8s.aws - names: - categories: - - karpenter - kind: AWSNodeTemplate - listKind: AWSNodeTemplateList - plural: awsnodetemplates - singular: awsnodetemplate - scope: Cluster - versions: - - name: v1alpha1 + - name: v1beta1 schema: openAPIV3Schema: - description: AWSNodeTemplate is the Schema for the AWSNodeTemplate API + description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: AWSNodeTemplateSpec is the top level specification for the - AWS Karpenter Provider. This will contain configuration necessary to - launch instances in AWS. + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. properties: amiFamily: description: AMIFamily is the AMI family that instances use. + enum: + - AL2 + - AL2023 + - Bottlerocket + - Ubuntu + - Custom + - Windows2019 + - Windows2022 type: string - amiSelector: - additionalProperties: - type: string - description: AMISelector discovers AMIs to be used by Amazon EC2 tags. - type: object - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in amiSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || + has(x.owner)))' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean blockDeviceMappings: description: BlockDeviceMappings to be applied to provisioned nodes. items: @@ -459,27 +936,38 @@ spec: volume is deleted on instance termination. type: boolean encrypted: - description: Encrypted indicates whether the EBS volume - is encrypted. Encrypted volumes can only be attached to - instances that support Amazon EBS encryption. If you are - creating a volume from a snapshot, you can't specify an - encryption value. + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. type: boolean iops: - description: "IOPS is the number of I/O operations per second - (IOPS). For gp3, io1, and io2 volumes, this represents - the number of IOPS that are provisioned for the volume. - For gp2 volumes, this represents the baseline performance - of the volume and the rate at which the volume accumulates - I/O credits for bursting. \n The following are the supported - values for each volume type: \n * gp3: 3,000-16,000 IOPS - \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n - For io1 and io2 volumes, we guarantee 64,000 IOPS only - for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 - IOPS. \n This parameter is supported for io1, io2, and - gp3 volumes only. This parameter is not supported for - gp2, st1, sc1, or standard volumes." + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. format: int64 type: integer kmsKeyID: @@ -490,131 +978,307 @@ spec: description: SnapshotID is the ID of an EBS snapshot type: string throughput: - description: 'Throughput to provision for a gp3 volume, - with a maximum of 1,000 MiB/s. Valid Range: Minimum value - of 125. Maximum value of 1000.' + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. format: int64 type: integer volumeSize: - anyOf: - - type: integer - - type: string - description: "VolumeSize in GiBs. You must specify either - a snapshot ID or a volume size. The following are the - supported volumes sizes for each volume type: \n * gp2 - and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 - and sc1: 125-16,384 \n * standard: 1-1,024" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string volumeType: - description: VolumeType of the block device. For more information, - see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 type: string type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean type: object + maxItems: 50 type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 context: - description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched type: boolean instanceProfile: - description: InstanceProfile is the AWS identity that instances use. - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. type: string - launchTemplate: - description: 'LaunchTemplateName for the node. If not specified, a - launch template will be generated. NOTE: This field is for specifying - a custom launch template and is exposed in the Spec as `launchTemplate` - for backwards compatibility.' + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 type: string metadataOptions: - description: "MetadataOptions for the generated launch template of - provisioned nodes. \n This specifies the exposure of the Instance - Metadata Service to provisioned EC2 nodes. For more information, - see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, - security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens required." + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. properties: httpEndpoint: - description: "HTTPEndpoint enables or disables the HTTP metadata - endpoint on provisioned nodes. If metadata options is non-nil, - but this parameter is not specified, the default state is \"enabled\". - \n If you specify a value of \"disabled\", instance metadata - will not be accessible on the node." + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled type: string httpProtocolIPv6: - description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint - for the instance metadata service on provisioned nodes. If metadata - options is non-nil, but this parameter is not specified, the - default state is "disabled". + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string httpPutResponseHopLimit: - description: HTTPPutResponseHopLimit is the desired HTTP PUT response - hop limit for instance metadata requests. The larger the number, - the further instance metadata requests can travel. Possible - values are integers from 1 to 64. If metadata options is non-nil, - but this parameter is not specified, the default value is 1. + default: 2 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. format: int64 + maximum: 64 + minimum: 1 type: integer httpTokens: - description: "HTTPTokens determines the state of token usage for - instance metadata requests. If metadata options is non-nil, - but this parameter is not specified, the default state is \"optional\". - \n If the state is optional, one can choose to retrieve instance - metadata with or without a signed token header on the request. - If one retrieves the IAM role credentials without a token, the - version 1.0 role credentials are returned. If one retrieves - the IAM role credentials using a valid signed token, the version - 2.0 role credentials are returned. \n If the state is \"required\", - one must send a signed token header with any instance metadata - retrieval requests. In this state, retrieving the IAM role credentials - always returns the version 2.0 credentials; the version 1.0 - credentials are not available." + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional type: string type: object - securityGroupSelector: - additionalProperties: - type: string - description: SecurityGroups specify the names of the security groups. - type: object - subnetSelector: - additionalProperties: - type: string - description: SubnetSelector discovers subnets by tags. A value of - "" is a wildcard. - type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of or security group + selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of or subnet selector terms. + The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' tags: additionalProperties: type: string description: Tags to be applied on ec2 resources like instances and launch templates. type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/managed-by + rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') userData: - description: UserData to be applied to the provisioned nodes. It must - be in the appropriate format based on the AMIFamily in use. Karpenter - will merge certain fields into this UserData to ensure nodes are - being provisioned with the correct configuration. + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. type: string + required: + - amiFamily + - securityGroupSelectorTerms + - subnetSelectorTerms type: object + x-kubernetes-validations: + - message: amiSelectorTerms is required when amiFamily == 'Custom' + rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != + 0 : true' + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) + && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. + You must delete and recreate this node class if you want to change + this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) + && has(self.instanceProfile)) status: - description: AWSNodeTemplateStatus contains the resolved state of the - AWSNodeTemplate + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: amis: - description: AMI contains the current AMI values that are available - to the cluster under the AMI selectors. + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: description: AMI contains resolved AMI selector values utilized for node launch @@ -629,29 +1293,29 @@ spec: description: Requirements of the AMI to be utilized on an instance type items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, the - values array must have a single element, which will - be interpreted as an integer. This array is replaced - during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator @@ -662,72 +1326,556 @@ spec: - requirements type: object type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile + for the role + type: string securityGroups: - description: SecurityGroups contains the current Security Groups values - that are available to the cluster under the SecurityGroups selectors. + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector + values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized + for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + +--- + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: nodeclaims.karpenter.sh +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 + conversionReviewVersions: + - v1beta1 + - v1 + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines + provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to + every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", + "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : + true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive + integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') + ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many + values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? + x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim + to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources + for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of + the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness items: - description: SecurityGroup contains resolved SecurityGroup selector - values utilized for node launch + description: Condition aliases the upstream type and adds additional + helper methods properties: - id: - description: ID of the security group + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string - name: - description: Name of the security group + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string - required: - - id - type: object - type: array - subnets: - description: Subnets contains the current Subnet values that are available - to the cluster under the subnet selectors. - items: - description: Subnet contains resolved Subnet selector values utilized - for node launch - properties: - id: - description: ID of the subnet + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ type: string - zone: - description: The associated availability zone + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - - id - - zone + - lastTransitionTime + - status + - type type: object type: array + imageID: + description: ImageID is an identifier for the image that runs on the + node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string type: object + required: + - spec type: object served: true storage: true subresources: status: {} - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/managed-by: kops - k8s-addon: karpenter.sh - name: machines.karpenter.sh -spec: - group: karpenter.sh - names: - categories: - - karpenter - kind: Machine - listKind: MachineList - plural: machines - singular: machine - scope: Cluster - versions: - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type @@ -748,134 +1896,168 @@ spec: name: Capacity priority: 1 type: string - - jsonPath: .metadata.labels.karpenter\.sh/provisioner-name - name: Provisioner + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool priority: 1 type: string - - jsonPath: .spec.machineTemplateRef.name - name: Template + - jsonPath: .spec.nodeClassRef.name + name: NodeClass priority: 1 type: string - name: v1alpha5 + name: v1beta1 schema: openAPIV3Schema: - description: Machine is the Schema for the Machines API + description: NodeClaim is the Schema for the NodeClaims API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: MachineSpec describes the desired state of the Machine + description: NodeClaimSpec describes the desired state of the NodeClaim properties: kubelet: - description: Kubelet are options passed to the kubelet when provisioning - nodes + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. items: type: string type: array - containerRuntime: - description: ContainerRuntime is the container runtime to be used - with your worker nodes. - type: string cpuCFSQuota: description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. type: boolean evictionHard: additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ type: string description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. format: int32 type: integer evictionSoft: additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ type: string description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionSoftGracePeriod: additionalProperties: type: string description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. format: int32 maximum: 100 minimum: 0 type: integer imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent format: int32 maximum: 100 minimum: 0 type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string description: KubeReserved contains resources reserved for Kubernetes system components. type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. format: int32 minimum: 0 type: integer podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. format: int32 minimum: 0 type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true + type: string description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - machineTemplateRef: - description: MachineTemplateRef is a reference to an object that defines + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration properties: apiVersion: @@ -891,37 +2073,98 @@ spec: - name type: object requirements: - description: Requirements are layered with Labels and applied to every - node. + description: Requirements are layered with GetLabels and applied to + every node. items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", + "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt type: string values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ type: array + x-kubernetes-list-type: atomic required: - key - operator type: object + maxItems: 100 type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : + true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive + integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') + ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many + values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? + x.values.size() >= x.minValues : true)' resources: - description: Resources models the resource requirements for the Machine + description: Resources models the resource requirements for the NodeClaim to launch properties: requests: @@ -932,36 +2175,44 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Requests describes the minimum required resources - for the Machine to launch + for the NodeClaim to launch type: object type: object startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute type: string key: description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ type: string required: - effect @@ -969,99 +2220,1378 @@ spec: type: object type: array taints: - description: Taints will be applied to the machine's node. + description: Taints will be applied to the NodeClaim's node. items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute type: string key: description: Required. The taint key to be applied to a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of + the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the + node + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + +--- + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: nodepools.karpenter.sh +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 + conversionReviewVersions: + - v1beta1 + - v1 + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + description: Disruption contains the parameters that relate to Karpenter's + disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for + disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + maxProperties: 100 + type: object + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || x.find("^([^/]+)").endsWith("node.kubernetes.io") || + x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") + || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that + defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied + to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") + || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] + || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value + defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a + single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == + ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= + 0) : true)' + - message: requirements with 'minValues' must have at least + that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string - value: - description: The taint value corresponding to the taint key. + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - - effect - - key + - lastTransitionTime + - message + - reason + - status + - type type: object type: array - type: object - status: - description: MachineStatus defines the observed state of Machine - properties: - allocatable: + resources: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of - the machine + description: Resources is the list of resources that have been provisioned. type: object - capacity: + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h + description: Disruption contains the parameters that relate to Karpenter's + disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenUnderutilized" if not specified + enum: + - WhenEmpty + - WhenUnderutilized + type: string + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + type: object + x-kubernetes-validations: + - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized + rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != + ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : + true' + - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty + rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) + : true' + limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the machine + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + maxProperties: 100 + type: object + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || x.find("^([^/]+)").endsWith("node.kubernetes.io") || + x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") + || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: NodeClaimSpec describes the desired state of the + NodeClaim + properties: + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + type: string + description: EvictionHard is the map of signal names to + quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + type: string + description: EvictionSoft is the map of signal names to + quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each + eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are + ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + type: string + description: KubeReserved contains resources reserved + for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + type: string + description: SystemReserved contains resources reserved + for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than + imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching + evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, + (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have + a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that + defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied + to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", + "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", + "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", + "topology.kubernetes.io/zone", "topology.kubernetes.io/region", + "node.kubernetes.io/instance-type", "kubernetes.io/arch", + "kubernetes.io/os", "node.kubernetes.io/windows-build"] + || self.find("^([^/]+)").endsWith("node.kubernetes.io") + || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") + || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") + || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] + || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", + "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", + "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", + "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", + "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", + "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", + "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", + "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", + "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", + "karpenter.k8s.aws/instance-accelerator-count"] + || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value + defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a + single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == + ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= + 0) : true)' + - message: requirements with 'minValues' must have at least + that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for + the NodeClaim to launch + maxProperties: 0 + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources + for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: Required. The taint key to be applied to + a node. + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + required: + - spec type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: conditions: description: Conditions contains signals for health and readiness items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + description: Condition aliases the upstream type and adds additional + helper methods properties: lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about - the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime + - message + - reason - status - type type: object type: array - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object type: object + required: + - spec type: object served: true - storage: true + storage: false subresources: status: {} @@ -1076,8 +3606,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1099,8 +3629,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1116,89 +3646,14 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-cert namespace: kube-system --- -apiVersion: v1 -data: - loglevel.webhook: debug - zap-logger-config: | - { - "level": "debug", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": ["stdout"], - "errorOutputPaths": ["stderr"], - "encoding": "console", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } -kind: ConfigMap -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: config-logging - namespace: kube-system - ---- - -apiVersion: v1 -data: - aws.assumeRoleARN: "" - aws.assumeRoleDuration: 15m - aws.clusterCABundle: "" - aws.clusterEndpoint: https://api.internal.minimal.example.com - aws.clusterName: minimal.example.com - aws.defaultInstanceProfile: "" - aws.enableENILimitedPodDensity: "true" - aws.enablePodENI: "false" - aws.interruptionQueueName: "" - aws.isolatedVPC: "false" - aws.vmMemoryOverheadPercent: "0.075" - batchIdleDuration: 1s - batchMaxDuration: 10s - featureGates.driftEnabled: "false" -kind: ConfigMap -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: karpenter-global-settings - namespace: kube-system - ---- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1208,8 +3663,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh rbac.authorization.k8s.io/aggregate-to-admin: "true" name: karpenter-admin @@ -1217,10 +3672,10 @@ rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status - - machines - - machines/status + - nodepools + - nodepools/status + - nodeclaims + - nodeclaims/status verbs: - get - list @@ -1231,7 +3686,7 @@ rules: - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates + - ec2nodeclasses verbs: - get - list @@ -1251,18 +3706,18 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-core rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status - - machines - - machines/status + - nodepools + - nodepools/status + - nodeclaims + - nodeclaims/status verbs: - get - list @@ -1285,6 +3740,7 @@ rules: resources: - storageclasses - csinodes + - volumeattachments verbs: - get - watch @@ -1300,10 +3756,9 @@ rules: - list - watch - apiGroups: - - admissionregistration.k8s.io + - apiextensions.k8s.io resources: - - validatingwebhookconfigurations - - mutatingwebhookconfigurations + - customresourcedefinitions verbs: - get - watch @@ -1319,8 +3774,8 @@ rules: - apiGroups: - karpenter.sh resources: - - machines - - machines/status + - nodeclaims + - nodeclaims/status verbs: - create - delete @@ -1329,8 +3784,8 @@ rules: - apiGroups: - karpenter.sh resources: - - provisioners - - provisioners/status + - nodepools + - nodepools/status verbs: - update - patch @@ -1346,9 +3801,9 @@ rules: resources: - nodes verbs: - - create - patch - delete + - update - apiGroups: - "" resources: @@ -1356,14 +3811,18 @@ rules: verbs: - create - apiGroups: - - admissionregistration.k8s.io - resourceNames: - - validation.webhook.karpenter.sh - - validation.webhook.config.karpenter.sh + - "" + resources: + - pods + verbs: + - delete +- apiGroups: + - apiextensions.k8s.io resources: - - validatingwebhookconfigurations + - customresourcedefinitions verbs: - update + - patch --- @@ -1376,40 +3835,24 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter rules: - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates + - ec2nodeclasses verbs: - get - list - watch -- apiGroups: - - admissionregistration.k8s.io - resourceNames: - - validation.webhook.karpenter.k8s.aws - resources: - - validatingwebhookconfigurations - verbs: - - update -- apiGroups: - - admissionregistration.k8s.io - resourceNames: - - defaulting.webhook.karpenter.k8s.aws - resources: - - mutatingwebhookconfigurations - verbs: - - update - apiGroups: - karpenter.k8s.aws resources: - - awsnodetemplates - - awsnodetemplates/status + - ec2nodeclasses + - ec2nodeclasses/status verbs: - patch - update @@ -1425,8 +3868,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-core roleRef: @@ -1449,8 +3892,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter roleRef: @@ -1473,8 +3916,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1490,7 +3933,6 @@ rules: - "" resources: - configmaps - - namespaces - secrets verbs: - get @@ -1504,25 +3946,10 @@ rules: - secrets verbs: - update -- apiGroups: - - "" - resourceNames: - - karpenter-global-settings - - config-logging - resources: - - configmaps - verbs: - - update - - patch - - delete - apiGroups: - coordination.k8s.io resourceNames: - karpenter-leader-election - - webhook.configmapwebhook.00-of-01 - - webhook.defaultingwebhook.00-of-01 - - webhook.validationwebhook.00-of-01 - - webhook.webhookcertificates.00-of-01 resources: - leases verbs: @@ -1534,12 +3961,6 @@ rules: - leases verbs: - create -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create --- @@ -1552,8 +3973,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-dns namespace: kube-system @@ -1578,8 +3999,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-lease namespace: kube-node-lease @@ -1610,8 +4031,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1635,8 +4056,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-dns namespace: kube-system @@ -1660,8 +4081,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter-lease namespace: kube-node-lease @@ -1685,17 +4106,21 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system spec: ports: - name: http-metrics - port: 8000 + port: 8080 protocol: TCP targetPort: http-metrics + - name: webhook-metrics + port: 8001 + protocol: TCP + targetPort: webhook-metrics - name: https-webhook port: 8443 protocol: TCP @@ -1716,8 +4141,8 @@ metadata: app.kubernetes.io/instance: karpenter app.kubernetes.io/managed-by: kops app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 + app.kubernetes.io/version: 1.0.0 + helm.sh/chart: karpenter-1.0.0 k8s-addon: karpenter.sh name: karpenter namespace: kube-system @@ -1726,7 +4151,8 @@ spec: revisionHistoryLimit: 10 selector: matchLabels: - karpenter: webhook + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter strategy: rollingUpdate: maxUnavailable: 1 @@ -1736,7 +4162,6 @@ spec: labels: app.kubernetes.io/instance: karpenter app.kubernetes.io/name: karpenter - karpenter: webhook kops.k8s.io/managed-by: kops spec: affinity: @@ -1776,8 +4201,14 @@ spec: value: karpenter - name: WEBHOOK_PORT value: "8443" + - name: WEBHOOK_METRICS_PORT + value: "8001" + - name: DISABLE_WEBHOOK + value: "false" + - name: LOG_LEVEL + value: debug - name: METRICS_PORT - value: "8000" + value: "8080" - name: HEALTH_PROBE_PORT value: "8081" - name: SYSTEM_NAMESPACE @@ -1790,13 +4221,27 @@ spec: containerName: controller divisor: "0" resource: limits.memory + - name: FEATURE_GATES + value: SpotToSpotConsolidation=false + - name: BATCH_MAX_DURATION + value: 10s + - name: BATCH_IDLE_DURATION + value: 1s + - name: CLUSTER_NAME + value: minimal.example.com + - name: CLUSTER_ENDPOINT + value: https://api.internal.minimal.example.com + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "0.075" + - name: RESERVED_ENIS + value: "0" - name: AWS_REGION value: us-test-1 - name: AWS_ROLE_ARN value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com - name: AWS_WEB_IDENTITY_TOKEN_FILE value: /var/run/secrets/amazonaws.com/token - image: public.ecr.aws/karpenter/controller:v0.31.3 + image: public.ecr.aws/karpenter/controller:1.0.0 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -1806,15 +4251,18 @@ spec: timeoutSeconds: 30 name: controller ports: - - containerPort: 8000 + - containerPort: 8080 name: http-metrics protocol: TCP - - containerPort: 8081 - name: http + - containerPort: 8001 + name: webhook-metrics protocol: TCP - containerPort: 8443 name: https-webhook protocol: TCP + - containerPort: 8081 + name: http + protocol: TCP readinessProbe: httpGet: path: /readyz @@ -1833,9 +4281,9 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsGroup: 65536 + runAsGroup: 65532 runAsNonRoot: true - runAsUser: 65536 + runAsUser: 65532 seccompProfile: type: RuntimeDefault volumeMounts: @@ -1845,7 +4293,7 @@ spec: dnsPolicy: Default priorityClassName: system-cluster-critical securityContext: - fsGroup: 10001 + fsGroup: 65532 serviceAccountName: karpenter tolerations: - key: node-role.kubernetes.io/master @@ -1874,174 +4322,8 @@ spec: --- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: defaulting.webhook.karpenter.k8s.aws -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: defaulting.webhook.karpenter.k8s.aws - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.karpenter.sh -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.karpenter.sh - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.config.karpenter.sh -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.config.karpenter.sh - objectSelector: - matchLabels: - app.kubernetes.io/part-of: karpenter - sideEffects: None - ---- - -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: karpenter.sh - app.kubernetes.io/instance: karpenter - app.kubernetes.io/managed-by: kops - app.kubernetes.io/name: karpenter - app.kubernetes.io/version: 0.31.3 - helm.sh/chart: karpenter-v0.31.3 - k8s-addon: karpenter.sh - name: validation.webhook.karpenter.k8s.aws -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: karpenter - namespace: kube-system - port: 8443 - failurePolicy: Fail - name: validation.webhook.karpenter.k8s.aws - rules: - - apiGroups: - - karpenter.k8s.aws - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - awsnodetemplates - - awsnodetemplates/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - operations: - - CREATE - - UPDATE - resources: - - provisioners - - provisioners/status - sideEffects: None - ---- - -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: creationTimestamp: null labels: @@ -2050,15 +4332,47 @@ metadata: k8s-addon: karpenter.sh name: karpenter-nodes-default spec: - launchTemplate: karpenter-nodes-default.minimal.example.com - subnetSelector: - kops.k8s.io/instance-group/karpenter-nodes-default: '*' - kubernetes.io/cluster/minimal.example.com: '*' + amiFamily: Custom + amiSelectorTerms: + - id: ami-12345678 + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: {} + rootVolume: true + instanceProfile: nodes.minimal.example.com + kubelet: + kubeReserved: + cpu: 500m + memory: 1G + maxPods: 50 + systemReserved: + cpu: 500m + memory: 1G + securityGroupSelectorTerms: + - tags: + KubernetesCluster: minimal.example.com + Name: nodes.minimal.example.com + kubernetes.io/cluster/minimal.example.com: owned + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/karpenter-nodes-default: '*' + kubernetes.io/cluster/minimal.example.com: '*' + tags: + KubernetesCluster: minimal.example.com + Name: karpenter-nodes-default.minimal.example.com + aws-node-termination-handler/managed: "" + k8s.io/cluster-autoscaler/node-template/label/foo: baz + k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name: karpenter-nodes-default + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: "" + k8s.io/role/node: "1" + kops.k8s.io/instancegroup: karpenter-nodes-default + kubernetes.io/cluster/minimal.example.com: owned +status: {} --- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1 +kind: NodePool metadata: creationTimestamp: null labels: @@ -2067,40 +4381,41 @@ metadata: k8s-addon: karpenter.sh name: karpenter-nodes-default spec: - consolidation: - enabled: true - kubeletConfiguration: - kubeReserved: - cpu: 500m - memory: 1G - maxPods: 50 - systemReserved: - cpu: 500m - memory: 1G - providerRef: - name: karpenter-nodes-default - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: - - spot - - on-demand - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: node.kubernetes.io/instance-type - operator: In - values: - - c5.large - startupTaints: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized + disruption: + consolidateAfter: 720h0m0s + template: + metadata: + labels: + foo: baz + spec: + expireAfter: 720h0m0s + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: karpenter-nodes-default + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - on-demand + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: node.kubernetes.io/instance-type + operator: In + values: + - c5.large + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized +status: {} --- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: creationTimestamp: null labels: @@ -2109,15 +4424,52 @@ metadata: k8s-addon: karpenter.sh name: karpenter-nodes-single-machinetype spec: - launchTemplate: karpenter-nodes-single-machinetype.minimal.example.com - subnetSelector: - kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*' - kubernetes.io/cluster/minimal.example.com: '*' + amiFamily: Custom + amiSelectorTerms: + - id: ami-12345678 + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + encrypted: true + iops: 4000 + kmsKeyID: arn:aws:kms:us-east-1:012345678910:key/1234abcd-12ab-34cd-56ef-1234567890ab + throughput: 200 + volumeSize: 200G + volumeType: gp3 + rootVolume: true + - deviceName: /dev/xvdd + ebs: + encrypted: true + kmsKeyID: arn:aws:kms:us-east-1:012345678910:key/1234abcd-12ab-34cd-56ef-1234567890ab + volumeSize: 20G + volumeType: gp2 + instanceProfile: nodes.minimal.example.com + kubelet: {} + securityGroupSelectorTerms: + - tags: + KubernetesCluster: minimal.example.com + Name: nodes.minimal.example.com + kubernetes.io/cluster/minimal.example.com: owned + subnetSelectorTerms: + - tags: + kops.k8s.io/instance-group/karpenter-nodes-single-machinetype: '*' + kubernetes.io/cluster/minimal.example.com: '*' + tags: + KubernetesCluster: minimal.example.com + Name: karpenter-nodes-single-machinetype.minimal.example.com + aws-node-termination-handler/managed: "" + k8s.io/cluster-autoscaler/node-template/label/foo: bar + k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name: karpenter-nodes-single-machinetype + k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node: "" + k8s.io/role/node: "1" + kops.k8s.io/instancegroup: karpenter-nodes-single-machinetype + kubernetes.io/cluster/minimal.example.com: owned +status: {} --- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1 +kind: NodePool metadata: creationTimestamp: null labels: @@ -2126,24 +4478,33 @@ metadata: k8s-addon: karpenter.sh name: karpenter-nodes-single-machinetype spec: - consolidation: - enabled: true - providerRef: - name: karpenter-nodes-single-machinetype - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: - - spot - - on-demand - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: node.kubernetes.io/instance-type - operator: In - values: - - t2.medium - startupTaints: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized + disruption: + consolidateAfter: 720h0m0s + template: + metadata: + labels: + foo: bar + spec: + expireAfter: 720h0m0s + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: karpenter-nodes-single-machinetype + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - on-demand + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: node.kubernetes.io/instance-type + operator: In + values: + - t2.medium + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized +status: {} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content index ff4b815fddac9..4ecad94f47d11 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content @@ -46,6 +46,7 @@ KubeletConfig: logLevel: 2 maxPods: 50 nodeLabels: + foo: baz karpenter.sh/provisioner-name: karpenter-nodes-default node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.9 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content index b3d1f81b489ca..193c156ce9fba 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content @@ -42,6 +42,7 @@ KubeletConfig: kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 nodeLabels: + foo: bar karpenter.sh/provisioner-name: karpenter-nodes-single-machinetype node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.9 diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 4ebefd3b9d986..0bd14c9f81104 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -594,6 +594,7 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "baz" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" @@ -607,6 +608,7 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "baz" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" @@ -618,6 +620,7 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "baz" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" @@ -633,12 +636,23 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp ebs { delete_on_termination = true encrypted = true - iops = 3000 - throughput = 125 - volume_size = 128 + iops = 4000 + kms_key_id = "arn:aws:kms:us-east-1:012345678910:key/1234abcd-12ab-34cd-56ef-1234567890ab" + throughput = 200 + volume_size = 200 volume_type = "gp3" } } + block_device_mappings { + device_name = "/dev/xvdd" + ebs { + delete_on_termination = true + encrypted = true + kms_key_id = "arn:aws:kms:us-east-1:012345678910:key/1234abcd-12ab-34cd-56ef-1234567890ab" + volume_size = 20 + volume_type = "gp2" + } + } iam_instance_profile { name = aws_iam_instance_profile.nodes-minimal-example-com.id } @@ -669,6 +683,7 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "bar" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" @@ -682,6 +697,7 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "bar" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" @@ -693,6 +709,7 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/foo" = "bar" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1"