Merge branch 'awslabs:main' into nodegroup

Author: hakuna-matatah

tests/pipelines/eks/awscli-cl2-load-with-addons.yaml

@@ -95,32 +95,32 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
-  - name: create-mng-nodes
+  - name: install-fluentbit-addon
     params:
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: desired-nodes
-      value: $(params.desired-nodes)
-    - name: host-cluster-node-role-name
-      value: $(params.cluster-name)-node-role
     - name: endpoint
       value: $(params.endpoint)
     runAfter:
     - create-mng-monitoring-nodes
     taskRef:
       kind: Task
-      name:  awscli-eks-nodegroup-create
-  - name: install-fluentbit-addon
+      name: eks-addon-fluentbit
+  - name: create-mng-nodes
     params:
     - name: cluster-name
       value: $(params.cluster-name)
+    - name: desired-nodes
+      value: $(params.desired-nodes)
+    - name: host-cluster-node-role-name
+      value: $(params.cluster-name)-node-role
     - name: endpoint
       value: $(params.endpoint)
     runAfter:
-    - create-mng-nodes
+    - install-fluentbit-addon
     taskRef:
       kind: Task
-      name: eks-addon-fluentbit
+      name:  awscli-eks-nodegroup-create
   - name: create-cw-agent-addon
     params:
     - name: cluster-name
@@ -147,7 +147,6 @@ spec:
     - name: amp-workspace-id
       value: '$(params.amp-workspace-id)'
     runAfter:
-    - install-fluentbit-addon
     - create-cw-agent-addon
     taskRef:
       kind: Task

tests/pipelines/eks/awscli-eks-cl2-load.yaml

@@ -81,6 +81,7 @@ spec:
     runAfter:
     - create-cluster-node-role
     - create-cluster-service-role
+    - awscli-vpc-create
     taskRef:
       kind: Task
       name:  awscli-eks-cluster-create-with-vpc-stack
@@ -107,32 +108,32 @@ spec:
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
-  - name: create-mng-nodes
+  - name: install-fluentbit-addon
     params:
     - name: cluster-name
       value: $(params.cluster-name)
-    - name: desired-nodes
-      value: $(params.desired-nodes)
-    - name: host-cluster-node-role-name
-      value: "$(params.cluster-name)-node-role"
     - name: endpoint
       value: $(params.endpoint)
     runAfter:
     - create-mng-monitoring-nodes
     taskRef:
       kind: Task
-      name:  awscli-eks-nodegroup-create
-  - name: install-fluentbit-addon
+      name: eks-addon-fluentbit
+  - name: create-mng-nodes
     params:
     - name: cluster-name
       value: $(params.cluster-name)
+    - name: desired-nodes
+      value: $(params.desired-nodes)
+    - name: host-cluster-node-role-name
+      value: "$(params.cluster-name)-node-role"
     - name: endpoint
       value: $(params.endpoint)
     runAfter:
-    - create-mng-nodes
+    - install-fluentbit-addon
     taskRef:
       kind: Task
-      name: eks-addon-fluentbit
+      name:  awscli-eks-nodegroup-create
   - name: generate
     params:
     - name: pods-per-node
@@ -152,7 +153,7 @@ spec:
     - name: amp-workspace-id
       value: '$(params.amp-workspace-id)'
     runAfter:
-    - install-fluentbit-addon
+    - create-mng-nodes
     taskRef:
       kind: Task
       name: load

tests/tasks/addons/fluentbit.yaml

@@ -48,4 +48,5 @@ spec:
       EOL
       kubectl apply -f ./fluentbit-configmap.yaml
       kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml
+      kubectl patch daemonset fluent-bit -n amazon-cloudwatch -p='{"spec":{"template": {"spec":  {"tolerations": [{"key": "monitoring", "operator": "Equal", "value": "true", "effect": "NoSchedule"}]}}}}'
       kubectl rollout status daemonset fluent-bit -n amazon-cloudwatch --timeout 5m

tests/tasks/generators/clusterloader/load.yaml

@@ -12,7 +12,7 @@ spec:
     default: https://github.com/kubernetes/perf-tests.git
   - name: cl2-branch
     description: "The branch of clusterloader2 you want to use"
-    default: "release-1.23"
+    default: "master"
   - name: nodes-per-namespace
     description: "nodes per namespace to get created for load test "
     default: "100"
@@ -75,22 +75,19 @@ spec:
       MEDIUM_STATEFUL_SETS_PER_NAMESPACE: 0
       # we are not testing PVS at this point
       CL2_ENABLE_PVS: false
-      PROMETHEUS_SCRAPE_KUBE_PROXY: true
+      PROMETHEUS_SCRAPE_KUBE_PROXY: false
       PROMETHEUS_SCRAPE_APISERVER_ONLY: true
       ENABLE_SYSTEM_POD_METRICS: false
       NODE_MODE: master 
+      CL2_DISABLE_DAEMONSETS: true
+      CL2_ALLOWED_SLOW_API_CALLS: 1000000
+      CL2_PROMETHEUS_NODE_SELECTOR: "eks.amazonaws.com/nodegroup: monitoring-$(params.cluster-name)-nodes-1"
       EOL
       cat $(workspaces.source.path)/overrides.yaml
       cp $(workspaces.source.path)/overrides.yaml $(workspaces.results.path)/overrides.yaml
       
       # Enable Prometheus if the remote workspace id is provided
       if [ -n "$(params.amp-workspace-id)" ]; then
-      # TODO: Currently pathing the prometheus manifests for remote write. Move this to upsteam going forward.
-      echo "volumeBindingMode: WaitForFirstConsumer" >> $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0ssd-storage-class.yaml
-      cat $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0ssd-storage-class.yaml
-      # TODO: Currently for v1.23 tests the node selector is hardcoded and can't be changed https://github.com/kubernetes/perf-tests/blob/release-1.23/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml#L15 via params. So for patching i am deleting the existing node selector and re-adding the updated node-selector later. Since this has been fixed in the current master https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml#L24 we can get rid of this going forward.
-      sed -i '/nodeSelector/d' $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
-      sed -i '/kubernetes.io\/os: linux/d' $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
       cat << EOF >> $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
         containers:
           - name: aws-sigv4-proxy-sidecar
@@ -116,14 +113,18 @@ spec:
         externalLabels:
           cluster_name: $(params.cluster-name)
           s3_path: $S3_RESULT_PATH
-        nodeSelector:
-          kubernetes.io/os: linux
-          eks.amazonaws.com/nodegroup: monitoring-$(params.cluster-name)-nodes-1
       EOF
       cat $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
+      cat << EOF >> $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0prometheus-operator-deployment.yaml
+            tolerations:
+              - key: monitoring
+                operator: Exists
+                effect: NoSchedule  
+      EOF
+      cat $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0prometheus-operator-deployment.yaml
       fi
   - name: run-loadtest
-    image: 197575167141.dkr.ecr.us-west-2.amazonaws.com/clusterloader2:latest
+    image: '197575167141.dkr.ecr.us-west-2.amazonaws.com/clusterloader2:a15645'
     onError: continue
     script: |
       #!/bin/bash
@@ -132,7 +133,7 @@ spec:
         ENDPOINT_FLAG="--endpoint $(params.endpoint)"
       fi
       if [ -n "$(params.amp-workspace-id)" ]; then
-        CL2_PROMETHEUS_FLAGS="--enable-prometheus-server=true --prometheus-storage-class-provisioner kubernetes.io/aws-ebs --prometheus-storage-class-volume-type gp2 --prometheus-manifest-path=$(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/"
+        CL2_PROMETHEUS_FLAGS="--enable-prometheus-server=true --prometheus-pvc-storage-class gp2 --prometheus-manifest-path=$(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/"
       fi
       aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
       cat $(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml

tests/tasks/setup/eks/awscli-cp-with-vpc.yaml

@@ -62,3 +62,5 @@ spec:
       kubectl set env daemonset/aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true
       # install csi drivers.
       kubectl apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=$(params.aws-ebs-csi-driver-version)"
+      # TODO: Calculate replicas based on the cluster size going forward.
+      kubectl scale --replicas 20 deploy coredns -n kube-system