Workaround konnectivity ingress issue

Signed-off-by: Alexey Makhov <[email protected]>
Signed-off-by: makhov <[email protected]>

Author: makhov

e2e/ingress_test.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	e2eutil "github.com/k0sproject/k0smotron/e2e/util"
+	podexec "github.com/k0sproject/k0smotron/internal/exec"
 	"github.com/k0sproject/k0smotron/internal/util"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -152,6 +153,14 @@ func ingressSupportSpec(t *testing.T) {
 		clusterv1.ClusterNameLabel: workloadClusterName,
 	}), "Should list machines")
 
+	podList := &corev1.PodList{}
+	err = bootstrapClusterProxy.GetClient().List(ctx, podList, client.InNamespace(testNamespace.Name))
+	require.NoError(t, err, "Should list k0smotron pods")
+	time.Sleep(10 * time.Second) // Wait a bit for konnectivity agent to have logs
+	out, err := podexec.PodExecCmdOutput(ctx, bootstrapClusterProxy.GetClientSet(), bootstrapClusterProxy.GetRESTConfig(), podList.Items[0].Name, testNamespace.Name, "k0s kc logs -n kube-system ds/konnectivity-agent")
+	require.NoError(t, err)
+	t.Logf("Konnectivity agent logs:\n%s", string(out))
+
 	for _, m := range machineList.Items {
 		var (
 			stdout bytes.Buffer

internal/controller/k0smotron.io/k0smotroncluster_configmap.go

@@ -281,7 +281,7 @@ func getV1Beta1Spec(kmc *km.Cluster, sans []string) map[string]interface{} {
 		}
 		v1beta1Spec["konnectivity"] = map[string]any{
 			"externalAddress": kmc.Spec.Ingress.KonnectivityHost,
-			"agentPort":       kmc.Spec.Ingress.Port,
+			"agentPort":       int64(kmc.Spec.Service.KonnectivityPort),
 		}
 	}
 

internal/controller/k0smotron.io/k0smotroncluster_ingress.go

@@ -29,6 +29,18 @@ func (scope *kmcScope) reconcileIngress(ctx context.Context, kmc *km.Cluster) er
 		return fmt.Errorf("failed to patch haproxy configmap for ingress: %w", err)
 	}
 
+	configMap, err = scope.generateKonnectivityIngressConfigMap(kmc)
+	if err != nil {
+		return fmt.Errorf("failed to generate ingress manifests configmap: %w", err)
+	}
+	_ = kcontrollerutil.SetExternalOwnerReference(kmc, &configMap, scope.client.Scheme(), scope.externalOwner)
+
+	err = scope.client.Patch(ctx, &configMap, client.Apply, patchOpts...)
+
+	if err != nil {
+		return fmt.Errorf("failed to patch haproxy configmap for ingress: %w", err)
+	}
+
 	var foundManifest bool
 	for _, manifest := range kmc.Spec.Manifests {
 		if manifest.Name == kmc.GetIngressManifestsConfigMapName() {
@@ -46,6 +58,15 @@ func (scope *kmcScope) reconcileIngress(ctx context.Context, kmc *km.Cluster) er
 					},
 				},
 			},
+		}, corev1.Volume{
+			Name: "konnectivity",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: kmc.GetIngressManifestsConfigMapName() + "-konnectivity",
+					},
+				},
+			},
 		})
 	}
 
@@ -173,6 +194,115 @@ spec:
 	return configMap, nil
 }
 
+func (scope *kmcScope) generateKonnectivityIngressConfigMap(kmc *km.Cluster) (corev1.ConfigMap, error) {
+	configMap := corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        kmc.GetIngressManifestsConfigMapName() + "-konnectivity",
+			Namespace:   kmc.Namespace,
+			Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc),
+		},
+		Data: map[string]string{
+			"konnectivity-agent.yaml": fmt.Sprintf(`apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:konnectivity-server
+  labels:
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: system:konnectivity-server
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: konnectivity-agent
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: konnectivity-agent
+  namespace: kube-system
+  name: konnectivity-agent
+spec:
+  selector:
+    matchLabels:
+      k8s-app: konnectivity-agent
+  template:
+    metadata:
+      labels:
+        k8s-app: konnectivity-agent
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '8093'
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        supplementalGroups: [0]
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      tolerations:
+        - operator: Exists
+      containers:
+        - image: quay.io/k0sproject/apiserver-network-proxy-agent:v0.33.0
+          imagePullPolicy: IfNotPresent
+          name: konnectivity-agent
+          command: ["/proxy-agent"]
+          env:
+              - name: NODE_IP
+                valueFrom:
+                  fieldRef:
+                    fieldPath: status.hostIP
+          args:
+            - --logtostderr=true
+            - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            - --proxy-server-host=%s
+            - --proxy-server-port=%d
+            - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
+            - --agent-identifiers=host=$(NODE_IP)
+            - --agent-id=$(NODE_IP)
+          volumeMounts:
+            - mountPath: /var/run/secrets/tokens
+              name: konnectivity-agent-token
+          livenessProbe:
+            httpGet:
+              port: 8093
+              path: /healthz
+            initialDelaySeconds: 15
+            timeoutSeconds: 15
+          readinessProbe:
+            httpGet:
+              port: 8093
+              path: /readyz
+            initialDelaySeconds: 15
+            timeoutSeconds: 15
+      serviceAccountName: konnectivity-agent
+      volumes:
+        - name: konnectivity-agent-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: konnectivity-agent-token
+                  audience: system:konnectivity-server`, kmc.Spec.Ingress.KonnectivityHost, kmc.Spec.Ingress.Port),
+		},
+	}
+
+	return configMap, nil
+}
+
 func (scope *kmcScope) generateIngress(kmc *km.Cluster) v1.Ingress {
 	annotations := kcontrollerutil.AnnotationsForK0smotronCluster(kmc)
 	if annotations == nil {