Skip to content

Commit 777d772

Browse files
cartermckinnoncartermckinnon
committed
Add unmanaged nodegroup support
1 parent cb3ee5c commit 777d772

File tree

4 files changed

+132
-29
lines changed

4 files changed

+132
-29
lines changed

‎tests/assets/asg_node_group.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
---
2+
AWSTemplateFormatVersion: '2010-09-09'
3+
Description: 'Unmanaged EKS nodegroup using EC2 AutoScaling'
4+
Parameters:
5+
AutoScalingGroupName:
6+
Description: Name of ASG.
7+
Type: String
8+
VpcId:
9+
Type: AWS::EC2::VPC::Id
10+
SubnetIds:
11+
Type: List<AWS::EC2::Subnet::Id>
12+
SecurityGroup:
13+
Type: AWS::EC2::SecurityGroup::Id
14+
LaunchTemplateName:
15+
Type: String
16+
Description: Launch template name.
17+
LaunchTemplateVersion:
18+
Type: String
19+
Description: Launch template version. Default is 0 as the launch template being used is generally ephemeral/single-use.
20+
Default: "0"
21+
NodeCount:
22+
Type: Number
23+
LaunchTemplateOverrides:
24+
Type: List<AWS::AutoScaling::AutoScalingGroup::LaunchTemplateOverrides>
25+
InstanceType:
26+
Description: EC2 instance type.
27+
Type: String
28+
Resources:
29+
AutoScalingGroup:
30+
Type: AWS::AutoScaling::AutoScalingGroup
31+
UpdatePolicy:
32+
AutoScalingRollingUpdate:
33+
WaitOnResourceSignals: true
34+
PauseTime: PT15M
35+
Properties:
36+
AutoScalingGroupName: !Ref AutoScalingGroupName
37+
DesiredCapacity: !Ref NodeCount
38+
MinSize: !Ref NodeCount
39+
MaxSize: !Ref NodeCount
40+
MixedInstancesPolicy:
41+
LaunchTemplate:
42+
LaunchTemplateSpecification:
43+
LaunchTemplateName: !Ref LaunchTemplateName
44+
Version: !Ref LaunchTemplateVersion
45+
Overrides: !Ref LaunchTemplateOverrides
46+
VPCZoneIdentifier:
47+
!Ref SubnetIds
48+
Tags:
49+
# necessary for kubelet's legacy, in-tree cloud provider
50+
- Key: !Sub kubernetes.io/cluster/${ClusterName}
51+
Value: owned
52+
PropagateAtLaunch: true

‎tests/assets/eks_node_group_launch_template_al2023.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,22 @@ Parameters:
2929
Type: String
3030
Description: Launch template ImageId value, which may be an AMI ID or resolve:ssm reference.
3131
Default: ''
32+
NodeRoleName:
33+
Type: String
34+
Description: Name of the IAM Role for the node instances.
35+
SecurityGroup:
36+
Type: AWS::EC2::SecurityGroup::Id
37+
Description: EKS-created cluster security group that allows node communication with the control plane.
3238
Conditions:
3339
AMIProvided:
3440
!Not [!Equals [!Ref AMI, '']]
3541
Resources:
42+
NodeInstanceProfile:
43+
Type: AWS::IAM::InstanceProfile
44+
Properties:
45+
Path: "/"
46+
Roles:
47+
- !Ref NodeRoleName
3648
LaunchTemplate:
3749
Type: AWS::EC2::LaunchTemplate
3850
Properties:
@@ -50,6 +62,10 @@ Resources:
5062
HttpPutResponseHopLimit: 2
5163
HttpEndpoint: enabled
5264
HttpTokens: required
65+
IamInstanceProfile:
66+
Arn: !GetAtt NodeInstanceProfile.Arn
67+
SecurityGroupIds:
68+
- !Ref SecurityGroup
5369
ImageId:
5470
!If
5571
- AMIProvided

‎tests/tasks/setup/eks/awscli-cfn-lt-al2023.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ spec:
8080
--arg CertificateAuthority "$(jq -r .cluster.certificateAuthority.data cluster.json)" \
8181
--arg KubeletConfig '$(params.kubelet-config)' \
8282
--arg AMI "$(params.ami)" \
83+
--arg SecurityGroup "$(jq -r .cluster.resourcesVpcConfig.clusterSecurityGroupId)" \
8384
'$ARGS.named | to_entries | map({"ParameterKey": .key, "ParameterValue": .value})' \
8485
> parameters.json
8586

‎tests/tasks/setup/eks/awscli-mng.yaml‎

Lines changed: 63 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ metadata:
66
namespace: scalability
77
spec:
88
description: |
9-
Create an EKS managed nodegroup for a given cluster.
10-
This Task can be used to create an EKS managed nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
9+
Create an EKS nodegroup, managed or unamaged, for a given cluster.
10+
This Task can be used to create an EKS managed or unmanaged nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
1111
params:
1212
- name: cluster-name
1313
description: The name of the EKS cluster you want to spin managed nodegroups for.
@@ -36,6 +36,9 @@ spec:
3636
- name: nodegroup-prefix
3737
description: Prefix that needs to be appended to asg names.
3838
default: ""
39+
- name: unmanaged-nodegroup-cfn-url
40+
default: ""
41+
description: URL for "unmanaged nodegroup" (AutoScaling group) CloudFormation template. If not specified, a managed nodegroup will be created.
3942
workspaces:
4043
- name: config
4144
mountPath: /config/
@@ -60,9 +63,12 @@ spec:
6063
TAINTS_FLAG="--taints $(params.host-taints)"
6164
fi
6265
63-
NG_SUBNETS=$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-cluster --name $(params.cluster-name) \
66+
NG_SUBNETS=$( \
6467
--query cluster.resourcesVpcConfig.subnetIds --output text \
6568
)
69+
70+
aws eks $ENDPOINT_FLAG --region $(params.region) describe-cluster --name $(params.cluster-name) --output json > cluster.json
71+
NG_SUBNETS=$(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(" ")' cluster.json)
6672
6773
max_nodes=$(params.max-nodes)
6874
nodes=$(params.desired-nodes)
@@ -73,32 +79,60 @@ spec:
7379
{
7480
node_group_name=$node_group-$1
7581
launch_template_name=$(params.cluster-name)-launchTemplate
76-
CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
77-
EC2_INSTANCES=$3
78-
if [ "$CREATED_NODEGROUP" == "" ]; then
79-
#create node group
80-
aws eks $ENDPOINT_FLAG create-nodegroup \
81-
--cluster-name $(params.cluster-name) \
82-
--nodegroup-name $node_group_name \
83-
--node-role $NODE_ROLE_ARN \
84-
--launch-template name=$launch_template_name\
85-
--region $(params.region) \
86-
--instance-types $EC2_INSTANCES \
87-
--scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
88-
--subnets $NG_SUBNETS $TAINTS_FLAG
82+
# if no unmanaged nodegroup cfn template is provided, assume we want managed nodegroups
83+
if [ "$(params.unmanaged-nodegroup-cfn-url)" = "" ]; then
84+
CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
85+
EC2_INSTANCES=$3
86+
if [ "$CREATED_NODEGROUP" == "" ]; then
87+
aws eks $ENDPOINT_FLAG create-nodegroup \
88+
--cluster-name $(params.cluster-name) \
89+
--nodegroup-name $node_group_name \
90+
--node-role $NODE_ROLE_ARN \
91+
--launch-template name=$launch_template_name\
92+
--region $(params.region) \
93+
--instance-types $EC2_INSTANCES \
94+
--scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
95+
--subnets $NG_SUBNETS $TAINTS_FLAG
96+
fi
97+
echo "CREATED_NODEGROUP=$node_group_name"
98+
while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
99+
do
100+
echo "$node_group_name is "CREATING" at $(date)"
101+
sleep 2
102+
done
103+
# TODO: do this for unmanaged nodes as well
104+
# right now we don't have an appropriate label to filter on for unmanaged nodes
105+
while true; do
106+
ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
107+
echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
108+
if [[ "$ready_node" -eq $2 ]]; then break; fi
109+
sleep 5
110+
done
111+
else
112+
STACK_NAME=$node_group_name
113+
STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text --region $(params.region))
114+
if [[ "$STACK_STATUS" == "" ]]; then
115+
curl -s $(params.unmanaged-nodegroup-cfn-url) -o ./cfn-template
116+
aws cloudformation create-stack \
117+
--region $(params.region)
118+
--stack-name $STACK_NAME \
119+
--template-body file://$(pwd)/cfn-template \
120+
--parameters ParameterKey=LaunchTemplateName,ParameterValue=$launch_template_name \
121+
ParameterKey=ClusterName,ParameterValue=$(params.cluster-name) \
122+
ParameterKey=SSHKeyName,ParameterValue=$SSH_KEY_NAME \
123+
ParameterKey=AutoScalingGroupName,ParameterValue=$node_group_name \
124+
ParameterKey=NodeCount,ParameterValue=$2 \
125+
ParameterKey=LaunchTemplateOverrides,ParameterValue=$(echo $EC2_INSTANCES | jq -R 'split(" ") | .[] | {"InstanceType":.}' | jq -s '.') \
126+
ParameterKey=SubnetIds,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(",")' cluster.json) \
127+
ParameterKey=SecurityGroup,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.clusterSecurityGroupId' cluster.json) \
128+
ParameterKey=VpcId,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.vpcId' cluster.json)
129+
130+
aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)
131+
echo "CREATED_CFN_STACK=$STACK_NAME"
132+
else
133+
echo "$STACK_NAME Already exists"
134+
fi
89135
fi
90-
echo "CREATED_NODEGROUP=$node_group_name"
91-
while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
92-
do
93-
echo "$node_group_name is "CREATING" at $(date)"
94-
sleep 2
95-
done
96-
while true; do
97-
ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
98-
echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
99-
if [[ "$ready_node" -eq $2 ]]; then break; fi
100-
sleep 5
101-
done
102136
}
103137
for i in $(seq 1 $asgs)
104138
do
@@ -119,4 +153,4 @@ spec:
119153
kubectl describe clusterrole eks:node-manager
120154
kubectl get nodes -o wide
121155
kubectl get ns
122-
kubectl get cs
156+
kubectl get cs

0 commit comments

Comments
 (0)