Updates Tasks to use kubeconfig from workspace. (#354)

This commit updates task to use kubeconfig from workspaces to make load
and addon related task generic.

Signed-off-by: Ashish Ranjan <[email protected]>

Author: ashishranjan738

tests/pipelines/eks/awscli-cl2-load-with-addons.yaml

@@ -82,6 +82,9 @@ spec:
     taskRef:
       kind: Task
       name: awscli-eks-cluster-create-with-vpc-stack
+    workspaces:
+    - name: config    
+      workspace: config
   - name: create-mng-monitoring-nodes
     params:
     - name: cluster-name
@@ -105,17 +108,21 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
   - name: install-fluentbit-addon
     params:
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: endpoint
-      value: $(params.endpoint)
     runAfter:
     - create-mng-monitoring-nodes
     taskRef:
       kind: Task
       name: eks-addon-fluentbit
+    workspaces:
+    - name: config    
+      workspace: config
   - name: create-mng-nodes
     params:
     - name: cluster-name
@@ -131,17 +138,21 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
   - name: create-cw-agent-addon
     params:
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: endpoint
-      value: $(params.endpoint)
     runAfter:
     - create-mng-nodes
     taskRef:
       kind: Task
       name:  eks-addon-cwagent
+    workspaces:
+    - name: config    
+      workspace: config
   - name: generate
     params:
     - name: cluster-name
@@ -158,8 +169,6 @@ spec:
       value: $(params.desired-nodes)
     - name: amp-workspace-id
       value: '$(params.amp-workspace-id)'
-    - name: endpoint
-      value: $(params.endpoint)
     runAfter:
     - create-cw-agent-addon
     taskRef:
@@ -170,6 +179,8 @@ spec:
       workspace: source
     - name: results
       workspace: results
+    - name: config    
+      workspace: config
   finally:
   - name: teardown
     params:
@@ -190,4 +201,5 @@ spec:
       name:  awscli-eks-cluster-teardown
   workspaces:
   - name: source
-  - name: results
+  - name: results
+  - name: config

tests/pipelines/eks/awscli-eks-cl2-load.yaml

@@ -85,6 +85,9 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-cluster-create-with-vpc-stack
+    workspaces:
+    - name: config    
+      workspace: config
   - name: create-mng-monitoring-nodes
     params:
     - name: cluster-name
@@ -108,17 +111,21 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
   - name: install-fluentbit-addon
     params:
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: endpoint
-      value: $(params.endpoint)
     runAfter:
     - create-mng-monitoring-nodes
     taskRef:
       kind: Task
       name: eks-addon-fluentbit
+    workspaces:
+    - name: config    
+      workspace: config
   - name: create-mng-nodes
     params:
     - name: cluster-name
@@ -134,6 +141,9 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
   - name: generate
     params:
     - name: pods-per-node
@@ -148,8 +158,6 @@ spec:
       value: $(params.desired-nodes)
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: endpoint
-      value: $(params.endpoint)
     - name: amp-workspace-id
       value: '$(params.amp-workspace-id)'
     runAfter:
@@ -161,7 +169,9 @@ spec:
     - name: source
       workspace: source
     - name: results
-      workspace: results  
+      workspace: results
+    - name: config    
+      workspace: config
   - name: cw-metrics
     params:
     - name: dimensions
@@ -195,4 +205,5 @@ spec:
       name:  awscli-eks-cluster-teardown
   workspaces:
   - name: source
-  - name: results
+  - name: results
+  - name: config

tests/tasks/addons/cwagent.yaml

@@ -13,19 +13,18 @@ spec:
   - name: region
     default: "us-west-2"
     description: The region where the cluster is in.
-  - name: endpoint
-    default: ""
+  workspaces:
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
   steps:
   - name: create-cw-agent
-    image: alpine/k8s:1.22.6
+    image: alpine/k8s:1.23.7
     script: |
-      echo "Approving KCM requests"
-      kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null || true
-      ENDPOINT_FLAG=""
-      if [ -n "$(params.endpoint)" ]; then
-        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
-      fi
-      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
+
       #kubectl commands are purely for knowing state of cluster before kicking off the test.
       kubectl version
       kubectl config current-context

tests/tasks/addons/fluentbit.yaml

@@ -13,19 +13,17 @@ spec:
   - name: region
     default: "us-west-2"
     description: The region where the cluster is in.
-  - name: endpoint
-    default: ""
+  workspaces:
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
   steps:
   - name: create-fluentbit-addon
-    image: alpine/k8s:1.22.6
+    image: alpine/k8s:1.23.7
     script: |
-      echo "Approving KCM requests"
-      kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null || true
-      ENDPOINT_FLAG=""
-      if [ -n "$(params.endpoint)" ]; then
-        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
-      fi
-      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
       #kubectl commands are purely for knowing state of cluster before kicking off the test.
       kubectl version
       kubectl config current-context

tests/tasks/generators/clusterloader/load.yaml

@@ -30,9 +30,6 @@ spec:
   - name: region
     default: "us-west-2"
     description: The region where the cluster is in.
-  - name: endpoint
-    default: ""
-    description: "aws eks enpoint to create clusters against"
   - name: cluster-name
     description: The name of the EKS cluster you want to spin.
   - name: amp-workspace-id
@@ -47,6 +44,12 @@ spec:
   - name: source
     mountPath: /src/k8s.io/
   - name: results
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
   steps:
   - name: git-clone      
     image: alpine/git
@@ -127,21 +130,16 @@ spec:
       cd $(workspaces.source.path)/perf-tests/clusterloader2/
       GOPROXY=direct GOOS=linux CGO_ENABLED=0  go build -v -o ./clusterloader ./cmd
   - name: run-loadtest
-    image: alpine/k8s:1.22.6
+    image: alpine/k8s:1.23.7
     onError: continue
     script: |
       #!/bin/bash
-      ENDPOINT_FLAG=""
-      if [ -n "$(params.endpoint)" ]; then
-        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
-      fi
       if [ -n "$(params.amp-workspace-id)" ]; then
         CL2_PROMETHEUS_FLAGS="--enable-prometheus-server=true --prometheus-pvc-storage-class gp2 --prometheus-manifest-path=$(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/"
       fi
-      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
       cat $(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml
       cd $(workspaces.source.path)/perf-tests/clusterloader2/
-      ENABLE_EXEC_SERVICE=false ./clusterloader --kubeconfig=/root/.kube/config --testconfig=$(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml --testoverrides=$(workspaces.source.path)/overrides.yaml --nodes=$(params.nodes) --provider=eks --report-dir=$(workspaces.results.path) --alsologtostderr --v=2 $CL2_PROMETHEUS_FLAGS
+      ENABLE_EXEC_SERVICE=false ./clusterloader --kubeconfig=$KUBECONFIG --testconfig=$(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml --testoverrides=$(workspaces.source.path)/overrides.yaml --nodes=$(params.nodes) --provider=eks --report-dir=$(workspaces.results.path) --alsologtostderr --v=2 $CL2_PROMETHEUS_FLAGS
       exit_code=$?
       if [ $exit_code -eq 0 ]; then
       echo "1" | tee $(results.datapoint.path)

tests/tasks/notifications/slack.yaml

@@ -15,10 +15,8 @@ spec:
     description: The message you want to post.
   steps:
   - name: send-notification
-    image: alpine/k8s:1.22.6
+    image: alpine/k8s:1.23.7
     script: |
-      echo "Approving KCM requests"
-      kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null || true
       if [ -n "$(params.slack-hook)" ]; then
         curl -H "Content-type: application/json" --data '{"Message": "$(params.slack-message)"}' -X POST  $(params.slack-hook)
       fi

tests/tasks/setup/eks/awscli-cp-with-vpc.yaml

@@ -27,12 +27,17 @@ spec:
   - name: aws-ebs-csi-driver-version
     default: release-1.13
     description: The release version for aws ebs csi driver.
+  workspaces:
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
   steps:
   - name: create-eks
-    image: alpine/k8s:1.23.13
+    image: alpine/k8s:1.23.7
     script: |
-      echo "Approving KCM requests"
-      kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null || true
       ENDPOINT_FLAG=""
       if [ -n "$(params.endpoint)" ]; then
         ENDPOINT_FLAG="--endpoint $(params.endpoint)"
@@ -53,7 +58,17 @@ spec:
         aws eks create-cluster --name $(params.cluster-name) --region $(params.region) --kubernetes-version $(params.kubernetes-version) --role-arn $SERVICE_ROLE_ARN --resources-vpc-config subnetIds=$subnets,securityGroupIds=$sg $ENDPOINT_FLAG
       fi
       aws eks $ENDPOINT_FLAG --region $(params.region) wait cluster-active --name $(params.cluster-name) 
+  - name: write-kubeconfig
+    image: alpine/k8s:1.23.7
+    script: |
+      ENDPOINT_FLAG=""
+      if [ -n "$(params.endpoint)" ]; then
+        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
+      fi
       aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
+  - name: install-addons-and-validate
+    image: alpine/k8s:1.23.7
+    script: |
       # enable PD on the cluster 
       kubectl api-versions 
       kubectl api-resources

tests/tasks/setup/eks/awscli-cp.yaml

@@ -25,14 +25,15 @@ spec:
     description: servicerole name to be used for eks cluster to perform operations in customer account to setup cluster
   workspaces:
   - name: config
-    description: |
-      A workspace into which a kubeconfig file called `kubeconfig` will be written that will contain the information required to access the cluster. The `kubeconfig` will expect to use [aws-iam-authenticator](https://github.com/kubernetes-sigs/aws-iam-authenticator/) to authenticate, so in order for it to be used it must be run in a container which contains both `kubectl` and `aws-iam-authenticator`.
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
   steps:
-  - name: write-kubeconfig
-    image: alpine/k8s:1.23.13
+  - name: create-cluster
+    image: alpine/k8s:1.23.7
     script: |
-      echo "Approving KCM requests"
-      kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null || true
       ENDPOINT_FLAG=""
       if [ -n "$(params.endpoint)" ]; then
         ENDPOINT_FLAG="--endpoint $(params.endpoint)"
@@ -51,8 +52,16 @@ spec:
       if [ "$CREATED_CLUSTER" == "" ]; then
         aws eks create-cluster --name $(params.cluster-name) --region $(params.region) --kubernetes-version $(params.kubernetes-version) --role-arn $SERVICE_ROLE_ARN --resources-vpc-config subnetIds=$subnets,securityGroupIds=$sg $ENDPOINT_FLAG
       fi
-      aws eks $ENDPOINT_FLAG --region $(params.region) wait cluster-active --name $(params.cluster-name) 
-      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
-      cp  /root/.kube/config $(workspaces.config.path)/kubeconfig
+  - name: write-kubeconfig
+    image: alpine/k8s:1.23.7
+    script: |
+      ENDPOINT_FLAG=""
+      if [ -n "$(params.endpoint)" ]; then
+        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
+      fi
+      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region) 
+  - name: update-cni-config
+    image: alpine/k8s:1.23.7
+    script: |
       # enable PD on the cluster 
       kubectl set env daemonset/aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true