feat(eksapi): enhance error messages for cfn failures

Author: mselim00

internal/deployers/eksapi/infra.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
 	"github.com/aws/aws-k8s-tester/internal/metrics"
+	"github.com/aws/aws-k8s-tester/internal/util"
 )
 
 const (
@@ -521,7 +522,7 @@ func (m *InfrastructureManager) createCloudWatchInfrastructureStack(clusterName
 		},
 	})
 	if err != nil {
-		return "", fmt.Errorf("failed to create CloudWatch infrastructure stack: %w", err)
+		return "", util.WrapCFNStackFailure(context.TODO(), m.clients.CFN(), fmt.Errorf("failed to create CloudWatch infrastructure stack: %w", err), stackName)
 	}
 
 	klog.Infof("waiting for CloudWatch infrastructure stack to be created: %s", *out.StackId)

internal/deployers/eksapi/node.go

@@ -28,6 +28,7 @@ import (
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
 	"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
+	"github.com/aws/aws-k8s-tester/internal/util"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
@@ -494,7 +495,7 @@ func (m *nodeManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *C
 	}
 	out, err := m.clients.CFN().CreateStack(context.TODO(), &input)
 	if err != nil {
-		return err
+		return util.WrapCFNStackFailure(context.TODO(), m.clients.CFN(), err, stackName)
 	}
 	klog.Infof("waiting for unmanaged nodegroup stack to be created: %s", aws.ToString(out.StackId))
 	err = cloudformation.NewStackCreateCompleteWaiter(m.clients.CFN()).

internal/util/cloudformation.go

@@ -0,0 +1,49 @@
+package util
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
+	types "github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
+)
+
+// TODO: implement AWS client wrappers, and incorporate this into the cfn:CreateStack call
+func WrapCFNStackFailure(ctx context.Context, cfnClient *cloudformation.Client, createStackErr error, stackName string) error {
+	if createStackErr == nil {
+		return createStackErr
+	}
+	resourceByFailureMode := make(map[string][]string)
+	eventsPaginator := cloudformation.NewDescribeStackEventsPaginator(cfnClient, &cloudformation.DescribeStackEventsInput{
+		StackName: &stackName,
+	})
+	for eventsPaginator.HasMorePages() {
+		page, err := eventsPaginator.NextPage(ctx)
+		if err != nil {
+			return createStackErr
+		}
+		for _, event := range page.StackEvents {
+			if event.ResourceStatus == types.ResourceStatusCreateFailed {
+				if _, ok := resourceByFailureMode[aws.ToString(event.ResourceStatusReason)]; !ok {
+					resourceByFailureMode[aws.ToString(event.ResourceStatusReason)] = []string{}
+				}
+				resourceByFailureMode[aws.ToString(event.ResourceStatusReason)] = append(resourceByFailureMode[aws.ToString(event.ResourceStatusReason)], aws.ToString(event.LogicalResourceId))
+			}
+		}
+	}
+	nonCancellationFailure := len(resourceByFailureMode) > 1
+	var enhancedDetails []string
+	for reason, resources := range resourceByFailureMode {
+		if nonCancellationFailure && reason == "Resource creation cancelled" {
+			// Ignore resource cancellation errors if there's another failure reported, those failures
+			// would just be a consequence of that failure. If all the failures are resource cancellation,
+			// then there was likely a user initiated delete of the whole stack based on a timeout
+			// waiting for one of the resources to create
+			continue
+		}
+		enhancedDetails = append(enhancedDetails, fmt.Sprintf("%s: %s", strings.Join(resources, ","), reason))
+	}
+	return fmt.Errorf("%w: %s", createStackErr, strings.Join(enhancedDetails, "--"))
+}