fix: add goroutine to handle remotemachine deletion when provisioning hangs

Signed-off-by: apedriza <[email protected]>

Author: apedriza

api/infrastructure/v1beta1/remote_machine_types.go

@@ -140,6 +140,9 @@ type RemoteMachineList struct {
 // +kubebuilder:subresource:status
 // +kubebuilder:metadata:labels="cluster.x-k8s.io/v1beta1=v1beta1"
 // +kubebuilder:metadata:labels="cluster.x-k8s.io/provider=infrastructure-k0smotron"
+// +kubebuilder:printcolumn:name="Address",type=string,JSONPath=".spec.machine.address",description="IP address or DNS name of the remote machine"
+// +kubebuilder:printcolumn:name="Reserved",type=string,JSONPath=".status.reserved",description="Indicates if the machine is reserved"
+// +kubebuilder:printcolumn:name="Remote Machine",type=string,JSONPath=".status.machineRef.name",description="Reference to the RemoteMachine"
 
 type PooledRemoteMachine struct {
 	metav1.TypeMeta   `json:",inline"`

config/clusterapi/infrastructure/bases/infrastructure.cluster.x-k8s.io_pooledremotemachines.yaml

@@ -17,7 +17,20 @@ spec:
     singular: pooledremotemachine
   scope: Namespaced
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - description: IP address or DNS name of the remote machine
+      jsonPath: .spec.machine.address
+      name: Address
+      type: string
+    - description: Indicates if the machine is reserved
+      jsonPath: .status.reserved
+      name: Reserved
+      type: string
+    - description: Reference to the RemoteMachine
+      jsonPath: .status.machineRef.name
+      name: Remote Machine
+      type: string
+    name: v1beta1
     schema:
       openAPIV3Schema:
         properties:

config/crd/bases/infrastructure/infrastructure.cluster.x-k8s.io_pooledremotemachines.yaml

@@ -17,7 +17,20 @@ spec:
     singular: pooledremotemachine
   scope: Namespaced
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - description: IP address or DNS name of the remote machine
+      jsonPath: .spec.machine.address
+      name: Address
+      type: string
+    - description: Indicates if the machine is reserved
+      jsonPath: .status.reserved
+      name: Reserved
+      type: string
+    - description: Reference to the RemoteMachine
+      jsonPath: .status.machineRef.name
+      name: Remote Machine
+      type: string
+    name: v1beta1
     schema:
       openAPIV3Schema:
         properties:

internal/controller/infrastructure/remote_machine_controller.go

@@ -19,6 +19,7 @@ package infrastructure
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"gopkg.in/yaml.v3"
 	v1 "k8s.io/api/core/v1"
@@ -33,6 +34,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	capiutil "sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/finalizers"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -90,6 +92,10 @@ func (r *RemoteMachineController) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, err
 	}
 
+	if finalizerAdded, err := finalizers.EnsureFinalizer(ctx, r.Client, rm, RemoteMachineFinalizer); err != nil || finalizerAdded {
+		return ctrl.Result{}, err
+	}
+
 	// Fetch the Machine that ows RemoteMachine
 	machine, err := capiutil.GetOwnerMachine(ctx, r.Client, rm.ObjectMeta)
 	if err != nil {
@@ -246,9 +252,25 @@ func (r *RemoteMachineController) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, nil
 	}
 
-	if !controllerutil.ContainsFinalizer(rm, RemoteMachineFinalizer) {
-		controllerutil.AddFinalizer(rm, RemoteMachineFinalizer)
-	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// Running a goroutine to monitor if the RemoteMachine gets deleted during provisioning. This way we can delete
+	// proceed to cleanup immediately without waiting for the provisioning to timeout. For example in scenarios where
+	// the bootstrap process hangs and the controller needs to be able to delete the RemoteMachine. Controller-runtime
+	// only runs one Reconcile at a time per object, so we need to monitor deletion in a separate goroutine.
+	go func() {
+		for {
+			updatedRemoteMachine := &infrastructure.RemoteMachine{}
+			if err := r.Get(ctx, client.ObjectKeyFromObject(rm), updatedRemoteMachine); err == nil &&
+				!updatedRemoteMachine.DeletionTimestamp.IsZero() {
+				log.Info("Cancelling Bootstrap because the underlying machine has been deleted")
+				cancel()
+				return
+			}
+			time.Sleep(5 * time.Second)
+		}
+	}()
 
 	defer func() {
 		log.Info("Reconcile complete")
@@ -287,6 +309,10 @@ func (r *RemoteMachineController) Reconcile(ctx context.Context, req ctrl.Reques
 			m.Labels[l] = rm.Labels[l]
 		}
 	}
+
+	if len(m.Annotations) == 0 {
+		m.Annotations = make(map[string]string)
+	}
 	for k := range rm.Annotations {
 		if _, ok := m.Annotations[k]; !ok {
 			m.Annotations[k] = rm.Annotations[k]

internal/controller/infrastructure/ssh_provisioner.go

@@ -104,26 +104,30 @@ func (p *SSHProvisioner) Provision(ctx context.Context) error {
 
 	// Write files first
 	for _, file := range p.cloudInit.Files {
+		p.log.Info("Uploading file", "path", file.Path, "permissions", file.Permissions)
 		if err := p.uploadFile(rigClient, file); err != nil {
 			return fmt.Errorf("failed to upload file: %w", err)
 		}
+		p.log.Info("Uploaded file", "path", file.Path, "permissions", file.Permissions)
 	}
 
 	if p.machine.Spec.CommandsAsScript {
 		// Run the install script
 		installScriptPath := filepath.Join(p.machine.Spec.WorkingDir, "k0s_install.sh")
+		p.log.Info("running install script", "command", installScriptPath)
 		output, err := rigClient.ExecOutput(installScriptPath)
 		if err != nil {
-			p.log.Error(err, "failed to run command", "output", output)
+			p.log.Error(err, "failed to run command", "command", installScriptPath, "output", output)
 			return fmt.Errorf("failed to run command: %w", err)
 		}
-		log.Info("executed command", "command", installScriptPath, "output", output)
+		log.Info("executed install script", "command", installScriptPath, "output", output)
 	} else {
 		// Run commands
 		for _, cmd := range p.cloudInit.Commands {
-			output, err := rigClient.ExecOutput(cmd)
+			p.log.Info("running command", "command", cmd)
+			output, err := rigClient.ExecOutputContext(ctx, cmd)
 			if err != nil {
-				p.log.Error(err, "failed to run command", "output", output)
+				p.log.Error(err, "failed to run command", "command", cmd, "output", output)
 				return fmt.Errorf("failed to run command: %w", err)
 			}
 			log.Info("executed command", "command", cmd, "output", output)
@@ -189,6 +193,7 @@ func (p *SSHProvisioner) Cleanup(ctx context.Context, mode RemoteMachineMode) er
 
 	if mode == ModeNonK0s {
 		// If k0s is not the bootstrap provider, we have nothing to do.
+		p.log.Info("k0smotron is not the bootstrap provider and no cleanup commands specified, skipping cleanup")
 		return nil
 	}
 
@@ -219,9 +224,21 @@ func (p *SSHProvisioner) Cleanup(ctx context.Context, mode RemoteMachineMode) er
 
 	p.log.Info("Cleaning up remote machine...")
 	for _, cmd := range cmds {
-		output, err := rigClient.ExecOutput(cmd)
+		output, err := rigClient.ExecOutputContext(ctx, cmd)
 		if err != nil {
-			p.log.Error(err, "failed to run command", "output", output)
+			// if k0s command is not installed, manually remove files added for k0s bootstrap.
+			if strings.Contains(err.Error(), "command not found") {
+				for _, file := range p.cloudInit.Files {
+					p.log.Info("Removing file", "path", file.Path)
+					err := rigClient.Sudo().FS().Remove(file.Path)
+					if err != nil {
+						p.log.Error(err, "failed to remove file", "path", file.Path)
+					} else {
+						p.log.Info("Removed file", "path", file.Path)
+					}
+				}
+			}
+			p.log.Error(err, "failed to run command", "command", cmd, "output", output)
 		}
 	}