Skip to content

Commit

Permalink
Add cluster scoped cni policies to node groups
Browse files Browse the repository at this point in the history
  • Loading branch information
@kishiel
kishiel committed Nov 1, 2023
1 parent f9ee5e6 commit ece5253
Show file tree
Hide file tree
Showing 3 changed files with 257 additions and 17 deletions.
3 changes: 2 additions & 1 deletion packages/@aws-cdk-testing/framework-integ/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
"experimentalDecorators": true,
"resolveJsonModule": true,
"composite": true,
"incremental": true
"incremental": true,
"esModuleInterop": true
},
"include": [
"config.ts",
Expand Down
229 changes: 227 additions & 2 deletions packages/aws-cdk-lib/aws-eks/lib/cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import * as iam from '../../aws-iam';
import * as kms from '../../aws-kms';
import * as lambda from '../../aws-lambda';
import * as ssm from '../../aws-ssm';
import { Annotations, CfnOutput, CfnResource, IResource, Resource, Stack, Tags, Token, Duration, Size } from '../../core';
import { Annotations, CfnJson, CfnOutput, CfnResource, IResource, Resource, Stack, Tags, Token, Duration, Size } from '../../core';

// defaults are based on https://eksctl.io
const DEFAULT_CAPACITY_COUNT = 2;
Expand Down Expand Up @@ -139,6 +139,13 @@ export interface ICluster extends IResource, ec2.IConnectable {
*/
readonly ipFamily?: IpFamily;

/**
* IAM Policy Statements which are used by the container network interface (CNI) and are scoped to the Cluster
*
* @see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md#scope-down-iam-policy-per-eks-cluster
*/
readonly cniPolicies?: iam.PolicyStatement[];

/**
* An AWS Lambda layer that contains the `aws` CLI.
*
Expand Down Expand Up @@ -1140,7 +1147,6 @@ abstract class ClusterBase extends Resource implements ICluster {
}

autoScalingGroup.role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKSWorkerNodePolicy'));
autoScalingGroup.role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy'));
autoScalingGroup.role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly'));

// EKS Required Tags
Expand Down Expand Up @@ -1434,6 +1440,13 @@ export class Cluster extends ClusterBase {

private readonly version: KubernetesVersion;

/**
* IAM Policy Statements which are used by the container network interface (CNI) and are scoped to the Cluster
*
* @see https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md#scope-down-iam-policy-per-eks-cluster
*/
public readonly cniPolicies?: iam.PolicyStatement[];

private readonly logging?: { [key: string]: [ { [key: string]: any } ] };

/**
Expand Down Expand Up @@ -1674,6 +1687,9 @@ export class Cluster extends ClusterBase {
commonCommandOptions.push(`--role-arn ${mastersRole.roleArn}`);
}

// Generate a cluster-scoped set of IAM policies for use with EKS CNI
this.cniPolicies = this.createClusterScopedCNIPolicies();

if (props.albController) {
this.albController = AlbController.create(this, { ...props.albController, cluster: this });
}
Expand All @@ -1697,6 +1713,7 @@ export class Cluster extends ClusterBase {
}

this.defineCoreDnsComputeType(props.coreDnsComputeType ?? CoreDnsComputeType.EC2);
this.defineClusterNameTag();

}

Expand Down Expand Up @@ -2024,6 +2041,214 @@ export class Cluster extends ClusterBase {
restorePatch: renderPatch(CoreDnsComputeType.EC2),
});
}

/**
* Patches the aws-node daemonset environment to define CLUSTER_NAME which
* will add the 'cluster.k8s.amazonaws.com/name' tag key and cluster name value to
* ENIs managed by the cluster. This tag enables conditional rules to be added
* to the node group role's IAM policies.
* https://github.com/aws/amazon-vpc-cni-k8s#cluster-name-tag
*/
private defineClusterNameTag() {

const patch = ({
spec: {
template: {
spec: {
containers: [
{
name: 'aws-node',
env: [{ name: 'CLUSTER_NAME', value: this.clusterName }],
},
],
},
},
},
});

new KubernetesPatch(this, 'ClusterNameTagPatch', {
cluster: this,
resourceName: 'daemonset/aws-node',
resourceNamespace: 'kube-system',
applyPatch: patch,
restorePatch: patch,
});
}

private createCNIConditions(scope: Construct, clusterName: string, vpc: ec2.IVpc): CniConditions {
// Use CfnJson to delay resolution to deployment-time
const requestTagClusterNameCondition = new CfnJson(scope, 'req-tag-clustername', {
value: {
['aws:RequestTag/cluster.k8s.amazonaws.com/name']: clusterName,
},
});

const resTagClusterNameOwned = new CfnJson(scope, 'res-tag-clustername-owned', {
value: {
[`aws:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
},
});

const resourceTagClusterNameCondition = new CfnJson(scope, 'res-tag-clustername', {
value: {
['aws:ResourceTag/cluster.k8s.amazonaws.com/name']: clusterName,
},
});

const resourceTagClusterNameConditionIPv6 = new CfnJson(scope, 'res-tag-clustername-ipv6', {
value: {
['aws:ResourceTag/cluster.k8s.amazonaws.com/name']: clusterName,
},
});

const vpcArnCondition = new CfnJson(scope, 'vpc-arn', {
value: {
['ec2:Vpc']: `arn:*:ec2:*:*:vpc/${vpc.vpcArn}`,
},
});

return {
requestTagClusterName: requestTagClusterNameCondition,
resourceTagClusterNameOwned: resTagClusterNameOwned,
resourceTagClusterName: resourceTagClusterNameCondition,
resourceTagClusterNameIPv6: resourceTagClusterNameConditionIPv6,
vpcArn: vpcArnCondition,
};
}

/**
* This function generates a cluster-scoped policy document for the EKS CNI Plugin which is based on
* https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md#scope-down-iam-policy-per-eks-cluster
* AWS partitions have been wildcarded
* @returns PolicyStatement[]
*/
private createClusterScopedCNIPolicies(): iam.PolicyStatement[] {

// Generate conditions for the container network interface IAM policies for the default Node controller
const cniConditions = this.createCNIConditions(this, this.clusterName, this.vpc);

let cniPolicyStatements: iam.PolicyStatement[] = [];

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['*'],
actions: [
'ec2:DescribeInstances',
'ec2:DescribeTags',
'ec2:DescribeNetworkInterfaces',
'ec2:DescribeInstanceTypes',
],
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:network-interface/*'],
actions: [
'ec2:CreateTags',
],
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:network-interface/*'],
actions: [
'ec2:CreateNetworkInterface',
],
conditions: {
StringEquals: cniConditions.requestTagClusterName,
},
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: [
'arn:*:ec2:*:*:subnet/*',
'arn:*:ec2:*:*:security-group/*',
],
actions: [
'ec2:CreateNetworkInterface',
],
conditions: {
ArnEquals: cniConditions.vpcArn,
},
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:network-interface/*'],
actions: [
'ec2:DeleteNetworkInterface',
'ec2:UnassignPrivateIpAddresses',
'ec2:AssignPrivateIpAddresses',
'ec2:AttachNetworkInterface',
'ec2:DetachNetworkInterface',
'ec2:ModifyNetworkInterfaceAttribute',
],
conditions: {
StringEquals: cniConditions.resourceTagClusterName,
},
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:network-interface/*'],
actions: [
'ec2:AttachNetworkInterface',
'ec2:DetachNetworkInterface',
'ec2:ModifyNetworkInterfaceAttribute',
],
conditions: {
StringEquals: cniConditions.resourceTagClusterNameOwned,
},
}));

cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:security-group/*'],
actions: [
'ec2:ModifyNetworkInterfaceAttribute',
],
}));

// Grant additional IPv6 networking permissions if running in IPv6
if (this.ipFamily == IpFamily.IP_V6) {
cniPolicyStatements.push(new iam.PolicyStatement({
resources: ['arn:*:ec2:*:*:network-interface/*'],
actions: [
'ec2:AssignIpv6Addresses',
'ec2:UnassignIpv6Addresses',
],
conditions: {
StringLike: cniConditions.resourceTagClusterNameIPv6,
},
}));
};

return cniPolicyStatements;
}
}

/**
* Condition statements for the container network interface IAM policies for the default Node controller
*/
export interface CniConditions {
/**
* RequestTag matching the cluster name
*/
readonly requestTagClusterName: CfnJson;

/**
* ResourceTag matching the cluster name with a value of 'owned'
*/
readonly resourceTagClusterNameOwned: CfnJson;

/**
* ResourceTag matching the cluster name
*/
readonly resourceTagClusterName: CfnJson;

/**
* ResourceTag matching the cluster name, is only used if the Cluster is IPv6
*/
readonly resourceTagClusterNameIPv6: CfnJson;

/**
* Arn comparison for the Cluster VPC
*/
readonly vpcArn: CfnJson;
}

/**
Expand Down
42 changes: 28 additions & 14 deletions packages/aws-cdk-lib/aws-eks/lib/managed-nodegroup.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { Construct, Node } from 'constructs';
import { Cluster, ICluster, IpFamily } from './cluster';
import { Cluster, ICluster } from './cluster';
import { CfnNodegroup } from './eks.generated';
import { InstanceType, ISecurityGroup, SubnetSelection, InstanceArchitecture, InstanceClass, InstanceSize } from '../../aws-ec2';
import { IRole, ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from '../../aws-iam';
import { IRole, ManagedPolicy, Role, ServicePrincipal } from '../../aws-iam';
import { IResource, Resource, Annotations, withResolved } from '../../core';

/**
Expand Down Expand Up @@ -253,7 +253,24 @@ export interface NodegroupOptions {
*
* @default - The latest available AMI version for the node group's current Kubernetes version is used.
*/

/**
* Limit the scope of the CNI, which uses the node group role, to the cluster
*
* @default true - The node group role will apply cluster-scoped permissions
*/
readonly applyLimitedCNIPoliciesToRole?: boolean;

/**
* The AMI version of the Amazon EKS optimized AMI to use with your node group (for example, `1.14.7- *YYYYMMDD*` ). By default, the latest available AMI version for the node group's current Kubernetes version is used. For more information, see [Amazon EKS optimized Linux AMI Versions](https://docs.aws.amazon.com/eks/latest/userguide/eks-linux-ami-versions.html) in the *Amazon EKS User Guide* .
*
* > Changing this value triggers an update of the node group if one is available. You can't update other properties at the same time as updating `Release Version` .
*
* @default - Latest AMI version
* @see http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-releaseversion
*/
readonly releaseVersion?: string;

/**
* The remote access (SSH) configuration to use with your node group. Disabled by default, however, if you
* specify an Amazon EC2 SSH key but do not specify a source security group when you create a managed node group,
Expand Down Expand Up @@ -355,6 +372,7 @@ export class Nodegroup extends Resource implements INodegroup {
private readonly desiredSize: number;
private readonly maxSize: number;
private readonly minSize: number;
private readonly applyLimitedCNIPoliciesToRole: boolean;

constructor(scope: Construct, id: string, props: NodegroupProps) {
super(scope, id, {
Expand All @@ -366,6 +384,7 @@ export class Nodegroup extends Resource implements INodegroup {
this.desiredSize = props.desiredSize ?? props.minSize ?? 2;
this.maxSize = props.maxSize ?? this.desiredSize;
this.minSize = props.minSize ?? 1;
this.applyLimitedCNIPoliciesToRole = props.applyLimitedCNIPoliciesToRole ?? true;

withResolved(this.desiredSize, this.maxSize, (desired, max) => {
if (desired === undefined) {return ;}
Expand Down Expand Up @@ -430,23 +449,18 @@ export class Nodegroup extends Resource implements INodegroup {
ngRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy'));
ngRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly'));

// Grant additional IPv6 networking permissions if running in IPv6
// https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html
if (props.cluster.ipFamily == IpFamily.IP_V6) {
ngRole.addToPrincipalPolicy(new PolicyStatement({
// eslint-disable-next-line @aws-cdk/no-literal-partition
resources: ['arn:aws:ec2:*:*:network-interface/*'],
actions: [
'ec2:AssignIpv6Addresses',
'ec2:UnassignIpv6Addresses',
],
}));
};
this.role = ngRole;
} else {
this.role = props.nodeRole;
}

// Apply the CNI policies to the node group role
if (props.cluster.cniPolicies && this.applyLimitedCNIPoliciesToRole) {
for (let policy of props.cluster.cniPolicies) {
this.role.addToPrincipalPolicy(policy);
}
}

this.validateUpdateConfig(props.maxUnavailable, props.maxUnavailablePercentage);

const resource = new CfnNodegroup(this, 'Resource', {
Expand Down

0 comments on commit ece5253

Please sign in to comment.