Add capability to create LT with customized parameters, fix make verify for substrate (#286)

* Split spec & status to support external calls

* Expose CreateAndAuthorizeSecurityGroupIngress

* Revert changes to split substrate spec & status, re-use operator asg & lt

* clean up

* Add ClusterCA into operator dataplane spec

* fix security group id in operator launch template

* review comments

Author: xdu31

operator/charts/kit-operator/crds/control-plane-crd.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.8.0
+    controller-gen.kubebuilder.io/version: v0.9.2
   creationTimestamp: null
   name: dataplanes.kit.k8s.sh
 spec:
@@ -40,10 +40,29 @@ spec:
                 description: AllocationStrategy helps user define the strategy to
                   provision worker nodes in EC2, defaults to "lowest-price"
                 type: string
+              amiID:
+                description: AmiID helps user create the launch template for work
+                  nodes If not provided, it's obtained by getting the recommended
+                  image id for the current k8s version
+                type: string
+              apiServerEndpoint:
+                description: APIServerEndpoint helps user create the launch template
+                  for work nodes If not provided, it's obtained from master instance
+                type: string
+              clusterCA:
+                description: ClusterCA helps user create the launch template for work
+                  nodes If not provided, get it from the current k8s cluster
+                format: byte
+                type: string
               clusterName:
                 description: ClusterName is used to connect the worker nodes to a
                   control plane clusterName.
                 type: string
+              instanceProfileName:
+                description: InstanceProfileName if provided is assigned to the kube
+                  nodes created If not provided, use the name for the current kit
+                  cluster
+                type: string
               instanceTypes:
                 description: InstanceTypes is an optional field thats lets user specify
                   the instance types for worker nodes, defaults to instance types
@@ -55,6 +74,16 @@ spec:
                 description: NodeCount is the desired number of worker nodes for this
                   dataplane.
                 type: integer
+              securityGroupSelector:
+                additionalProperties:
+                  type: string
+                description: SecurityGroupSelector lets user define label key and
+                  values for kit to select the security group for worker nodes. It
+                  can contain key:value to select security group with particular label,
+                  or a specific key:"*" to select all security group with a specific
+                  key. If no selector is provided, security group is discovered with
+                  control plane nodes
+                type: object
               subnetSelector:
                 additionalProperties:
                   type: string
@@ -110,9 +139,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

operator/charts/kit-operator/crds/data-plane-crd.yaml

@@ -58,4 +58,27 @@ type DataPlaneSpec struct {
 	// defaults to "lowest-price"
 	// +optional
 	AllocationStrategy string `json:"allocationStrategy,omitempty"`
+
+	// SecurityGroupSelector lets user define label key and values for kit to select the security group
+	// for worker nodes. It can contain key:value to select security group with particular label,
+	// or a specific key:"*" to select all security group with a specific key.
+	// If no selector is provided, security group is discovered with control plane nodes
+	// +optional
+	SecurityGroupSelector map[string]string `json:"securityGroupSelector,omitempty"`
+	// APIServerEndpoint helps user create the launch template for work nodes
+	// If not provided, it's obtained from master instance
+	// +optional
+	APIServerEndpoint string `json:"apiServerEndpoint,omitempty"`
+	// AmiID helps user create the launch template for work nodes
+	// If not provided, it's obtained by getting the recommended image id for the current k8s version
+	// +optional
+	AmiID string `json:"amiID,omitempty"`
+	// InstanceProfileName if provided is assigned to the kube nodes created
+	// If not provided, use the name for the current kit cluster
+	// +optional
+	InstanceProfileName string `json:"instanceProfileName,omitempty"`
+	// ClusterCA helps user create the launch template for work nodes
+	// If not provided, get it from the current k8s cluster
+	// +optional
+	ClusterCA []byte `json:"clusterCA,omitempty"`
 }

operator/pkg/apis/dataplane/v1alpha1/dataplane.go

@@ -89,25 +89,49 @@ func (c *Controller) deleteLaunchTemplate(ctx context.Context, templateName stri
 }
 
 func (c *Controller) createLaunchTemplate(ctx context.Context, dataplane *v1alpha1.DataPlane) error {
-	// Currently, we get the same security group assigned to control plane instances
-	// At some point, we will be creating dataplane specific security groups
-	securityGroupID, err := securitygroup.New(c.ec2api, c.kubeclient).For(ctx, dataplane.Spec.ClusterName)
-	if err != nil {
-		return fmt.Errorf("getting security group for control plane nodes, %w", err)
+	var (
+		securityGroupID string
+		err             error
+	)
+	if len(dataplane.Spec.SecurityGroupSelector) != 0 {
+		securityGroupID, err = c.securityGroupForSelector(ctx, dataplane.Spec.SecurityGroupSelector)
+		if err != nil {
+			return fmt.Errorf("getting security groups by selector, %w", err)
+		}
+	} else {
+		// Currently, we get the same security group assigned to control plane instances
+		// At some point, we will be creating dataplane specific security groups
+		securityGroupID, err = securitygroup.New(c.ec2api, c.kubeclient).For(ctx, dataplane.Spec.ClusterName)
+		if err != nil {
+			return fmt.Errorf("getting security group for control plane nodes, %w", err)
+		}
 	}
-	clusterEndpoint, err := master.GetClusterEndpoint(ctx, c.kubeclient, types.NamespacedName{Namespace: dataplane.Namespace, Name: dataplane.Spec.ClusterName})
-	if err != nil {
-		return fmt.Errorf("getting cluster endpoint, %w", err)
+	apiServerEndpoint := dataplane.Spec.APIServerEndpoint
+	if apiServerEndpoint == "" {
+		apiServerEndpoint, err = master.GetClusterEndpoint(ctx, c.kubeclient, types.NamespacedName{Namespace: dataplane.Namespace, Name: dataplane.Spec.ClusterName})
+		if err != nil {
+			return fmt.Errorf("getting cluster endpoint, %w", err)
+		}
 	}
-	caSecret, err := keypairs.Reconciler(c.kubeclient).GetSecretFromServer(ctx,
-		object.NamespacedName(master.RootCASecretNameFor(dataplane.Spec.ClusterName), dataplane.Namespace))
-	if err != nil {
-		return fmt.Errorf("getting control plane ca certificate, %w", err)
+	amiID := dataplane.Spec.AmiID
+	if amiID == "" {
+		amiID, err = c.amiID(ctx, dataplane)
+		if err != nil {
+			return fmt.Errorf("getting ami id for worker nodes, %w", err)
+		}
 	}
-	_, clusterCA := secrets.Parse(caSecret)
-	amiID, err := c.amiID(ctx, dataplane)
-	if err != nil {
-		return fmt.Errorf("getting ami id for worker nodes, %w", err)
+	instanceProfile := dataplane.Spec.InstanceProfileName
+	if len(instanceProfile) == 0 {
+		instanceProfile = iam.KitNodeInstanceProfileNameFor(dataplane.Spec.ClusterName)
+	}
+	clusterCA := dataplane.Spec.ClusterCA
+	if len(clusterCA) == 0 {
+		caSecret, err := keypairs.Reconciler(c.kubeclient).GetSecretFromServer(ctx,
+			object.NamespacedName(master.RootCASecretNameFor(dataplane.Spec.ClusterName), dataplane.Namespace))
+		if err != nil {
+			return fmt.Errorf("getting control plane ca certificate, %w", err)
+		}
+		_, clusterCA = secrets.Parse(caSecret)
 	}
 	input := &ec2.CreateLaunchTemplateInput{
 		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
@@ -123,22 +147,48 @@ func (c *Controller) createLaunchTemplate(ctx context.Context, dataplane *v1alph
 			InstanceType: ptr.String("t2.xlarge"), // TODO get this from dataplane spec
 			ImageId:      ptr.String(amiID),
 			IamInstanceProfile: &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{
-				Name: aws.String(iam.KitNodeInstanceProfileNameFor(dataplane.Spec.ClusterName)),
+				Name: aws.String(instanceProfile),
 			},
 			Monitoring:       &ec2.LaunchTemplatesMonitoringRequest{Enabled: ptr.Bool(true)},
 			SecurityGroupIds: []*string{ptr.String(securityGroupID)},
 			UserData: ptr.String(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(userData,
-				dataplane.Spec.ClusterName, dnsClusterIP, base64.StdEncoding.EncodeToString(clusterCA), clusterEndpoint)))),
+				dataplane.Spec.ClusterName, dnsClusterIP, base64.StdEncoding.EncodeToString(clusterCA), apiServerEndpoint)))),
 		},
 		LaunchTemplateName: ptr.String(TemplateName(dataplane.Spec.ClusterName)),
 		TagSpecifications:  generateEC2Tags("launch-template", dataplane.Spec.ClusterName),
 	}
-	if _, err := c.ec2api.CreateLaunchTemplate(input); err != nil {
+	if _, err := c.ec2api.CreateLaunchTemplateWithContext(ctx, input); err != nil {
 		return fmt.Errorf("creating launch template, %w", err)
 	}
 	return nil
 }
 
+func (c *Controller) securityGroupForSelector(ctx context.Context, selector map[string]string) (string, error) {
+	filters := []*ec2.Filter{}
+	// Filter by selector
+	for key, value := range selector {
+		if value == "*" {
+			filters = append(filters, &ec2.Filter{
+				Name:   aws.String("tag-key"),
+				Values: []*string{aws.String(key)},
+			})
+		} else {
+			filters = append(filters, &ec2.Filter{
+				Name:   aws.String(fmt.Sprintf("tag:%s", key)),
+				Values: []*string{aws.String(value)},
+			})
+		}
+	}
+	output, err := c.ec2api.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{Filters: filters})
+	if err != nil {
+		return "", fmt.Errorf("describing security groups %+v, %w", filters, err)
+	}
+	if len(output.SecurityGroups) != 1 {
+		return "", fmt.Errorf("wrong number of security groups by selector, %d", len(output.SecurityGroups))
+	}
+	return aws.StringValue(output.SecurityGroups[0].GroupId), nil
+}
+
 func (c *Controller) amiID(ctx context.Context, dataplane *v1alpha1.DataPlane) (string, error) {
 	kubeVersion, err := c.desiredKubernetesVersion(ctx, dataplane)
 	if err != nil {

operator/pkg/apis/dataplane/v1alpha1/zz_generated.deepcopy.go

@@ -17,6 +17,7 @@ package main
 import (
 	"os"
 	"time"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/awslabs/kubernetes-iteration-toolkit/substrate/pkg/apis/v1alpha1"
 	"github.com/awslabs/kubernetes-iteration-toolkit/substrate/pkg/controller/substrate"

operator/pkg/awsprovider/launchtemplate/reconciler.go

@@ -3,7 +3,7 @@ module github.com/awslabs/kubernetes-iteration-toolkit/substrate
 go 1.17
 
 require (
-	github.com/aws/aws-sdk-go v1.43.8
+	github.com/aws/aws-sdk-go v1.43.16
 	github.com/awslabs/kubernetes-iteration-toolkit/operator v0.0.0-20220818192233-b51328aa42d4
 	github.com/imdario/mergo v0.3.12
 	github.com/mitchellh/hashstructure/v2 v2.0.2

substrate/cmd/kitctl/bootstrap.go

@@ -162,8 +162,6 @@ github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.38.69/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.43.8 h1:8a/M9C4l5CxFNM6IuNx4F1p+ITJEX12VxWxUQo61cbc=
-github.com/aws/aws-sdk-go v1.43.8/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.43.16 h1:Y7wBby44f+tINqJjw5fLH3vA+gFq4uMITIKqditwM14=
 github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/awslabs/kubernetes-iteration-toolkit/operator v0.0.0-20220818192233-b51328aa42d4 h1:FRTo4S/JXaf7X1jhFQ4CUPYTvdC18XoQIRa/cORCaCM=

substrate/go.mod

@@ -106,8 +106,14 @@ func (c *Config) Create(ctx context.Context, substrate *v1alpha1.Substrate) (rec
 		return reconcile.Result{}, fmt.Errorf("generating authenticator config, %w", err)
 	}
 	// upload to s3 bucket
-	if err := c.S3Uploader.UploadWithIterator(ctx, NewDirectoryIterator(
-		aws.StringValue(discovery.Name(substrate)), path.Join(c.clusterConfigPath, aws.StringValue(discovery.Name(substrate))))); err != nil {
+	directoyIterator, err := NewDirectoryIterator(
+		aws.StringValue(discovery.Name(substrate)),
+		path.Join(c.clusterConfigPath, aws.StringValue(discovery.Name(substrate))),
+	)
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("building directory iterator %w", err)
+	}
+	if err := c.S3Uploader.UploadWithIterator(ctx, directoyIterator); err != nil {
 		return reconcile.Result{}, fmt.Errorf("uploading to S3 %w", err)
 	}
 	logging.FromContext(ctx).Debugf("Uploaded cluster configuration to s3://%s", aws.StringValue(discovery.Name(substrate)))
@@ -369,22 +375,25 @@ type DirectoryIterator struct {
 }
 
 // NewDirectoryIterator builds a new DirectoryIterator
-func NewDirectoryIterator(bucket, dir string) s3manager.BatchUploadIterator {
+func NewDirectoryIterator(bucket, dir string) (s3manager.BatchUploadIterator, error) {
 	var paths []string
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	walkFunc := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !info.IsDir() {
 			paths = append(paths, path)
 		}
 		return nil
-	})
+	}
+	if err := filepath.Walk(dir, walkFunc); err != nil {
+		return nil, err
+	}
 	return &DirectoryIterator{
 		filePaths: paths,
 		bucket:    bucket,
 		localDir:  dir,
-	}
+	}, nil
 }
 
 // Next returns whether next file exists or not