feat (Karpenter Add-on): v0.35 update, Node Role custom policies, and other fixes

examples/blueprint-construct/index.ts

@@ -14,6 +14,8 @@ const rikerManifestDir = './examples/teams/team-riker/';
 const teamManifestDirList = [burnhamManifestDir, rikerManifestDir];
 const blueprintID = 'blueprint-construct-dev';
 
+const karpenterNodeRolePoliciesDir = './examples/policies/karpenter/';
+
 export interface BlueprintConstructProps {
     /**
      * Id
@@ -148,6 +150,7 @@ export default class BlueprintConstruct {
                 nodePoolSpec: nodePoolSpec,
                 ec2NodeClassSpec: nodeClassSpec,
                 interruptionHandling: true,
+                nodeRoleAdditionalPoliciesDir: karpenterNodeRolePoliciesDir
             }),
             new blueprints.addons.AwsNodeTerminationHandlerAddOn(),
             new blueprints.addons.KubeviousAddOn(),

examples/policies/karpenter/node-role-efs-policy.json

@@ -0,0 +1,14 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "elasticfilesystem:ClientWrite",
+            "Resource": "*",
+            "Condition": {
+                "DateGreaterThan": {"aws:CurrentTime": "2024-02-01T00:00:00Z"},
+                "DateLessThan": {"aws:CurrentTime": "2024-06-30T23:59:59Z"}
+            }
+        }
+    ]
+}

examples/policies/karpenter/node-role-s3-policy.json

@@ -0,0 +1,14 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:ListBucket",
+            "Resource": "*",
+            "Condition": {
+                "DateGreaterThan": {"aws:CurrentTime": "2024-02-01T00:00:00Z"},
+                "DateLessThan": {"aws:CurrentTime": "2024-06-30T23:59:59Z"}
+            }
+        }
+    ]
+}

lib/addons/karpenter/index.ts

@@ -14,6 +14,13 @@ import { Rule } from 'aws-cdk-lib/aws-events';
 import { SqsQueue } from 'aws-cdk-lib/aws-events-targets';
 import { Cluster, KubernetesVersion } from 'aws-cdk-lib/aws-eks';
 
+export const DEFAULT_KARPENTER_NODE_ROLE_POLICIES = [
+    iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEKSWorkerNodePolicy"),
+    iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEC2ContainerRegistryReadOnly"),
+    iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore"),
+    iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEKS_CNI_Policy")
+];
+
 const versionMap: Map<KubernetesVersion, string> = new Map([
     [KubernetesVersion.V1_29, '0.34.0'],
     [KubernetesVersion.V1_28, '0.31.0'],
@@ -253,6 +260,13 @@ export interface KarpenterAddOnProps extends HelmAddOnUserProps {
      */
     ec2NodeClassSpec?: Ec2NodeClassSpec,
 
+    /**
+     * Optional for Karpenter Node Role additional policies, as users may require other policies in order for pods on Karpenter nodes to execute its tasks
+     * This is a reference to the directory where the IAM Policy JSON files are located
+     * NOTE: This addon will not validate the JSON files, and invalid files may cause deployment errors
+     */
+    nodeRoleAdditionalPoliciesDir?: string,
+
     /**
      * Flag for enabling Karpenter's native interruption handling 
      */
@@ -268,7 +282,7 @@ const RELEASE = 'blueprints-addon-karpenter';
 const defaultProps: HelmAddOnProps = {
     name: KARPENTER,
     namespace: KARPENTER,
-    version: 'v0.34.1',
+    version: '0.35.1',
     chart: KARPENTER,
     release: KARPENTER,
     repository: 'oci://public.ecr.aws/karpenter/karpenter',
@@ -302,6 +316,7 @@ export class KarpenterAddOn extends HelmAddOn {
         const version = this.options.version!;
 
         const interruption = this.options.interruptionHandling || false;
+        const nodeRoleAdditionalPoliciesDir = this.options.nodeRoleAdditionalPoliciesDir;
 
         // NodePool variables
         const labels = this.options.nodePoolSpec?.labels || {};
@@ -324,7 +339,7 @@ export class KarpenterAddOn extends HelmAddOn {
         const amiFamily = this.options.ec2NodeClassSpec?.amiFamily;
         const amiSelector = this.options.ec2NodeClassSpec?.amiSelector || {};
         const amiSelectorTerms = this.options.ec2NodeClassSpec?.amiSelectorTerms;
-        const instanceStorePolicy = this.options.ec2NodeClassSpec?.instanceStorePolicy || "";
+        const instanceStorePolicy = this.options.ec2NodeClassSpec?.instanceStorePolicy;
         const userData = this.options.ec2NodeClassSpec?.userData || "";
         const instanceProf = this.options.ec2NodeClassSpec?.instanceProfile; 
         const tags = this.options.ec2NodeClassSpec?.tags || {};
@@ -345,7 +360,7 @@ export class KarpenterAddOn extends HelmAddOn {
             this.options.ec2NodeClassSpec, amiFamily);
 
         // Set up the node role and instance profile
-        const [karpenterNodeRole, karpenterInstanceProfile] = this.setUpNodeRole(cluster, stackName, region);
+        const [karpenterNodeRole, karpenterInstanceProfile] = this.setUpNodeRole(clusterInfo, stackName, nodeRoleAdditionalPoliciesDir, region);
 
         // Create the controller policy
         let karpenterPolicyDocument;
@@ -686,40 +701,57 @@ export class KarpenterAddOn extends HelmAddOn {
      * Outputs to CloudFormation and map the role to the aws-auth ConfigMap
      * @param cluster EKS Cluster
      * @param stackName Name of the stack
+     * @param nodeRoleAdditionalPoliciesDir Directory containing additional policies to be attached to the Node Role
      * @param region Region of the stack
      * @returns [karpenterNodeRole, karpenterInstanceProfile]
      */
-    private setUpNodeRole(cluster: Cluster, stackName: string, region: string): [iam.Role, iam.CfnInstanceProfile] {
+    private setUpNodeRole(clusterInfo: ClusterInfo, stackName: string, nodeRoleAdditionalPoliciesDir: string | undefined, region: string): [iam.Role, iam.CfnInstanceProfile] {
+
+        const cluster = clusterInfo.cluster as Cluster;
+
         // Set up Node Role
-        const karpenterNodeRole = new iam.Role(cluster, 'karpenter-node-role', {
-            assumedBy: new iam.ServicePrincipal(`ec2.${cluster.stack.urlSuffix}`),
-            managedPolicies: [
-                iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEKSWorkerNodePolicy"),
-                iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEKS_CNI_Policy"),
-                iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEC2ContainerRegistryReadOnly"),
-                iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore"),
-            ],
+        const karpenterNodeRole = new iam.Role(clusterInfo.cluster, 'karpenter-node-role', {
+            assumedBy: new iam.ServicePrincipal(`ec2.${clusterInfo.cluster.stack.urlSuffix}`),
+            managedPolicies: DEFAULT_KARPENTER_NODE_ROLE_POLICIES,
             //roleName: `KarpenterNodeRole-${name}` // let role name to be generated as unique
         });
 
+        // TODO: Add EKS CNI Policy only if VPC CNI is not part of the provisioned addons as VPC CNI adds this particular policy
+        // if (!clusterInfo.getProvisionedAddOn('VpcCniAddOn')){
+        //     karpenterNodeRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonEKS_CNI_Policy"));
+        // }
+
+        // Generate PolicyDocuments if additional policies directory is provided
+        // Then attach the policies to the Karpenter Node Role
+        if (nodeRoleAdditionalPoliciesDir) {
+            const policyDocuments = utils.createPolicyDocuments(nodeRoleAdditionalPoliciesDir);
+            // Add any additional custom policies provided by the user
+            policyDocuments.forEach((policyDocument, index) => {
+                const policy = new iam.Policy(clusterInfo.cluster, 'karpenter-additional-policy-'+index, {
+                    document: policyDocument,
+                });
+                policy.attachToRole(karpenterNodeRole);
+            });
+        }
+
         // Set up Instance Profile
         const instanceProfileName = md5.Md5.hashStr(stackName+region);
-        const karpenterInstanceProfile = new iam.CfnInstanceProfile(cluster, 'karpenter-instance-profile', {
+        const karpenterInstanceProfile = new iam.CfnInstanceProfile(clusterInfo.cluster, 'karpenter-instance-profile', {
             roles: [karpenterNodeRole.roleName],
             instanceProfileName: `KarpenterNodeInstanceProfile-${instanceProfileName}`,
             path: '/'
         });
 
-        const clusterId = Names.uniqueId(cluster);
+        const clusterId = Names.uniqueId(clusterInfo.cluster);
 
         //Cfn output for Node Role in case of needing to add additional policies
-        new CfnOutput(cluster.stack, 'Karpenter Instance Node Role', {
+        new CfnOutput(clusterInfo.cluster.stack, 'Karpenter Instance Node Role', {
             value: karpenterNodeRole.roleName,
             description: "Karpenter add-on Node Role name",
             exportName: clusterId+"KarpenterNodeRoleName",
         });
         //Cfn output for Instance Profile for creating additional provisioners
-        new CfnOutput(cluster.stack, 'Karpenter Instance Profile name', { 
+        new CfnOutput(clusterInfo.cluster.stack, 'Karpenter Instance Profile name', { 
             value: karpenterInstanceProfile ? karpenterInstanceProfile.instanceProfileName! : "none",
             description: "Karpenter add-on Instance Profile name",
             exportName: clusterId+"KarpenterInstanceProfileName", 

lib/utils/yaml-utils.ts

@@ -1,8 +1,27 @@
 import * as eks from 'aws-cdk-lib/aws-eks';
 import { KubernetesManifest } from 'aws-cdk-lib/aws-eks';
+import { PolicyDocument } from 'aws-cdk-lib/aws-iam';
 import * as fs from 'fs';
 import * as yaml from 'js-yaml';
 
+/**
+ * Creates a list of PolicyDocuments for every JSON file in a directory
+ * @param dir Directory path to the JSON files.
+ * @returns List of PolicyDocument objects.
+ */
+export function createPolicyDocuments(dir: string): PolicyDocument[] {
+    const policyDocuments: PolicyDocument[] = [];
+    fs.readdirSync(dir, { encoding: 'utf8' }).forEach((file, _) => {
+        if (file.split('.').pop() == 'json') {
+            const data = fs.readFileSync(dir + file, 'utf8');
+            if (data != undefined) {
+                const policyDocument = PolicyDocument.fromJson(JSON.parse(data));
+                policyDocuments.push(policyDocument);
+            }
+        }
+    });
+    return policyDocuments;
+}
 
 /**
  * Applies all manifests from a directory. Note: The manifests are not checked, 

test/karpenter.test.ts

@@ -427,5 +427,39 @@ describe('Unit tests for Karpenter addon', () => {
         expect(manifest.spec.blockDeviceMappings).toBeDefined();
         expect(manifest.spec.blockDeviceMappings.length).toEqual(1);
         expect(manifest.spec.blockDeviceMappings[0]).toMatchObject(blockDeviceMapping);
-      });
+    });
+
+    test("Stack creation succeeds with node role additional policy", async () => {
+        const app = new cdk.App();
+
+        const blueprint = blueprints.EksBlueprint.builder();
+
+        const stack = await blueprint
+            .version(KubernetesVersion.V1_29)
+            .account("123567891")
+            .region("us-west-1")
+            .addOns(new blueprints.KarpenterAddOn({
+                version: '0.35.1',
+                nodeRoleAdditionalPoliciesDir: './test/karpenter/',
+            })).buildAsync(app, "stack-with-additional-node-role-policies");
+
+        let numAdditionalPolicies: number = 0;
+        const template = Template.fromStack(stack);
+        const policyResources = template.findResources("AWS::IAM::Policy");
+        const additionalNodeRolePolicies = Object.values(policyResources).find((policyResource) => {
+            if (policyResource?.Properties?.PolicyName) {
+                const policyName = policyResource.Properties.PolicyName;
+                if (typeof policyName === "string" && policyName.includes('karpenteradditionalpolicy')) { 
+                    numAdditionalPolicies++;
+                    return true;
+                }
+            }
+            return false;
+        });
+        expect(numAdditionalPolicies).toEqual(1);
+        const policyStatement = additionalNodeRolePolicies?.Properties?.PolicyDocument.Statement;
+        expect(policyStatement[0].Action).toEqual('s3:ListBucket');
+        expect(policyStatement[0].Effect).toEqual('Allow');
+        expect(policyStatement[0].Resource).toEqual('*');
+    });
 });

test/karpenter/policy.json

@@ -0,0 +1,10 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:ListBucket",
+            "Resource": "*"
+        }
+    ]
+}