implement e2e test for lock feature

Signed-off-by: Yuji Ito <[email protected]>

Author: llamerada-jp

test/e2e/e2e_suite_test.go

@@ -74,6 +74,7 @@ var _ = Describe("Fin", func() {
 	Context("wait environment", waitEnvironment)
 	Context("full backup", Label("full-backup"), Ordered, fullBackupTestSuite)
 	Context("incremental backup", Label("incremental-backup"), Ordered, incrementalBackupTestSuite)
+	Context("lock", Label("lock"), Label("misc"), Ordered, lockTestSuite)
 	Context("verification", Label("verification"), Label("misc"), Ordered, verificationTestSuite)
 	Context("delete incremental backup", Label("delete-incremental-backup"), Label("misc"), Ordered,
 		deleteIncrementalBackupTestSuite)

test/e2e/full_backup_test.go

@@ -195,7 +195,8 @@ func fullBackupTestSuite() {
 
 		VerifyNonExistenceOfRawImage(pvc, nodes[0])
 		VerifyDeletionOfJobsForBackup(ctx, k8sClient, finbackup)
-		VerifyDeletionOfSnapshotInFinBackup(ctx, ctrlClient, finbackup)
+		err = VerifyDeletionOfSnapshotInFinBackup(ctx, finbackup)
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	// CSATEST-1547
@@ -325,7 +326,8 @@ func fullBackupTestSuite() {
 			err = WaitForFinBackupDeletion(ctx, ctrlClient, fb, 2*time.Minute)
 			Expect(err).NotTo(HaveOccurred())
 			VerifyDeletionOfJobsForBackup(ctx, k8sClient, fb)
-			VerifyDeletionOfSnapshotInFinBackup(ctx, ctrlClient, fb)
+			err = VerifyDeletionOfSnapshotInFinBackup(ctx, fb)
+			Expect(err).NotTo(HaveOccurred())
 		}
 		VerifyNonExistenceOfRawImage(pvc, nodes[0])
 		VerifyNonExistenceOfRawImage(pvc, nodes[1])

test/e2e/incremental_test.go

@@ -225,7 +225,8 @@ func incrementalBackupTestSuite() {
 		Expect(rawImageData).To(Equal(dataOnIncrementalBackup), "Data in raw.img does not match the expected data")
 
 		VerifyDeletionOfJobsForBackup(ctx, k8sClient, finbackup1)
-		VerifyDeletionOfSnapshotInFinBackup(ctx, ctrlClient, finbackup1)
+		err = VerifyDeletionOfSnapshotInFinBackup(ctx, finbackup1)
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	// Description:

test/e2e/lock_test.go

@@ -0,0 +1,89 @@
+package e2e
+
+import (
+	"encoding/json"
+	"time"
+
+	finv1 "github.com/cybozu-go/fin/api/v1"
+	"github.com/cybozu-go/fin/internal/model"
+	"github.com/cybozu-go/fin/test/utils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func lockTestSuite() {
+	var ns *corev1.Namespace
+	var pvc *corev1.PersistentVolumeClaim
+	var fb *finv1.FinBackup
+	dummyLockID := "dummy-lock-id"
+	var poolName, imageName string
+
+	It("should setup environment", func(ctx SpecContext) {
+		ns = NewNamespace(utils.GetUniqueName("ns-"))
+		err := CreateNamespace(ctx, k8sClient, ns)
+		Expect(err).NotTo(HaveOccurred())
+
+		pvc = CreateBackupTargetPVC(ctx, k8sClient, ns, "Filesystem", rookStorageClass, "ReadWriteOnce", "100Mi")
+	})
+
+	It("should lock the volume", func(ctx SpecContext) {
+		pv, err := GetPvByPvc(ctx, k8sClient, pvc)
+		Expect(err).NotTo(HaveOccurred())
+		poolName = pv.Spec.CSI.VolumeAttributes["pool"]
+		imageName = pv.Spec.CSI.VolumeAttributes["imageName"]
+
+		// locked
+		_, _, err = kubectl("exec", "-n", rookNamespace, "deployment/"+finDeploymentName, "--",
+			"rbd", "-p", poolName, "lock", "add", imageName, dummyLockID)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should create backup and wait for the log", func(ctx SpecContext) {
+		fb, err := NewFinBackup(rookNamespace, utils.GetUniqueName("fb-"), pvc, nodes[0])
+		Expect(err).NotTo(HaveOccurred())
+		err = CreateFinBackup(ctx, ctrlClient, fb)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = WaitControllerLog(ctx,
+			"the volume is locked by another process.*"+string(fb.GetUID()),
+			3*time.Minute)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("checks that the snapshot is not created", func(ctx SpecContext) {
+		snapshots, err := ListRBDSnapshots(ctx, poolName, imageName)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(snapshots).To(BeEmpty())
+	})
+
+	It("should unlock the volume", func() {
+		stdout, _, err := kubectl("exec", "-n", rookNamespace, "deployment/"+finDeploymentName, "--",
+			"rbd", "-p", poolName, "--format", "json", "lock", "ls", imageName)
+		Expect(err).NotTo(HaveOccurred())
+		var locks []*model.RBDLock
+		err = json.Unmarshal(stdout, &locks)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(locks).To(HaveLen(1))
+
+		// unlock
+		_, _, err = kubectl("exec", "-n", rookNamespace, "deployment/"+finDeploymentName, "--",
+			"rbd", "-p", poolName, "lock", "rm", imageName, dummyLockID, locks[0].Locker)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should resume backup creation and complete it", func(ctx SpecContext) {
+		err := WaitForFinBackupStoredToNodeAndVerified(ctx, ctrlClient, fb, 1*time.Minute)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should not exist locks after backup completion", func() {
+		stdout, _, err := kubectl("exec", "-n", rookNamespace, "deployment/"+finDeploymentName, "--",
+			"rbd", "-p", poolName, "--format", "json", "lock", "ls", imageName)
+		Expect(err).NotTo(HaveOccurred())
+		var locks []*model.RBDLock
+		err = json.Unmarshal(stdout, &locks)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(locks).To(HaveLen(0))
+	})
+}

test/e2e/util_test.go

@@ -1,16 +1,20 @@
 package e2e
 
 import (
+	"bufio"
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	finv1 "github.com/cybozu-go/fin/api/v1"
+	"github.com/cybozu-go/fin/internal/model"
 	"github.com/cybozu-go/fin/test/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -26,6 +30,7 @@ import (
 )
 
 const (
+	finDeploymentName      = "fin-controller-manager"
 	rookNamespace          = "rook-ceph"
 	rookStorageClass       = "rook-ceph-block"
 	poolName               = "rook-ceph-block-pool"
@@ -80,7 +85,7 @@ func checkDeploymentReady(namespace, name string) error {
 func waitEnvironment() {
 	It("wait for fin-controller to be ready", func() {
 		Eventually(func() error {
-			return checkDeploymentReady(rookNamespace, "fin-controller-manager")
+			return checkDeploymentReady(rookNamespace, finDeploymentName)
 		}).Should(Succeed())
 	})
 }
@@ -455,6 +460,58 @@ func WaitForPVCDeletion(ctx context.Context, k8sClient kubernetes.Interface, pvc
 	})
 }
 
+// WaitControllerLog waits until the controller log matches the given pattern or the duration is exceeded.
+func WaitControllerLog(ctx SpecContext, pattern string, duration time.Duration) error {
+	GinkgoHelper()
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, duration)
+	defer cancel()
+
+	matcher := regexp.MustCompile(pattern)
+
+	command := exec.CommandContext(timeoutCtx, "kubectl", "logs", "-n", rookNamespace, "deployment/"+finDeploymentName, "-f")
+	stdoutPipe, err := command.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	err = command.Start()
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		_ = command.Process.Kill()
+		_ = command.Wait()
+	}()
+
+	// read stdout line by line until the pattern is found
+	scanner := bufio.NewScanner(stdoutPipe)
+	found := make(chan struct{})
+	go func() {
+		for scanner.Scan() {
+			select {
+			case <-timeoutCtx.Done():
+				return
+			default:
+			}
+			line := scanner.Text()
+			if matcher.MatchString(line) {
+				close(found)
+				return
+			}
+		}
+		if scanner.Err() != nil {
+			panic(scanner.Err())
+		}
+	}()
+
+	select {
+	case <-timeoutCtx.Done():
+		return timeoutCtx.Err()
+	case <-found:
+		return nil
+	}
+}
+
 func VerifySizeOfRestorePVC(ctx context.Context, c client.Client, restore *finv1.FinRestore) {
 	GinkgoHelper()
 
@@ -679,6 +736,30 @@ func GetNodeNames(ctx context.Context, k8sClient kubernetes.Interface) ([]string
 	return nodeNames, nil
 }
 
+func GetPvByPvc(ctx context.Context, k8sClient kubernetes.Interface, pvc *corev1.PersistentVolumeClaim) (*corev1.PersistentVolume, error) {
+	GinkgoHelper()
+
+	return k8sClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
+}
+
+func ListRBDSnapshots(ctx context.Context, poolName, imageName string) ([]*model.RBDSnapshot, error) {
+	GinkgoHelper()
+
+	stdout, stderr, err := kubectl("exec", "-n", rookNamespace, "deploy/rook-ceph-tools", "--",
+		"rbd", "snap", "ls", "-p", poolName, imageName, "--format", "json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to list RBD snapshots. stdout: %s, stderr: %s, err: %w",
+			string(stdout), string(stderr), err)
+	}
+
+	var snapshots []*model.RBDSnapshot
+	if err := json.Unmarshal(stdout, &snapshots); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal RBD snapshot list. err: %w", err)
+	}
+
+	return snapshots, nil
+}
+
 func VerifyRawImage(pvc *corev1.PersistentVolumeClaim, node string, expected []byte) {
 	GinkgoHelper()
 
@@ -706,13 +787,22 @@ func VerifyDeletionOfJobsForBackup(ctx context.Context, client kubernetes.Interf
 	Expect(err).NotTo(HaveOccurred(), "Deletion job should be deleted.")
 }
 
-func VerifyDeletionOfSnapshotInFinBackup(ctx context.Context, ctrlClient client.Client, finbackup *finv1.FinBackup) {
+func VerifyDeletionOfSnapshotInFinBackup(ctx context.Context, finbackup *finv1.FinBackup) error {
 	GinkgoHelper()
 
-	rbdImage := finbackup.Annotations["fin.cybozu.io/backup-target-rbd-image"]
-	stdout, stderr, err := kubectl("exec", "-n", rookNamespace, "deploy/rook-ceph-tools", "--",
-		"rbd", "info", fmt.Sprintf("%s/%s@fin-backup-%s", poolName, rbdImage, finbackup.UID))
-	Expect(err).To(HaveOccurred(), "Snapshot should be deleted. stdout: %s, stderr: %s", stdout, stderr)
+	imageName := finbackup.Annotations["fin.cybozu.io/backup-target-rbd-image"]
+	snapshots, err := ListRBDSnapshots(ctx, poolName, imageName)
+	if err != nil {
+		return err
+	}
+
+	expectedSnapName := fmt.Sprintf("fin-backup-%s", finbackup.UID)
+	for _, snapshot := range snapshots {
+		if snapshot.Name == expectedSnapName {
+			return fmt.Errorf("snapshot %s still exists", expectedSnapName)
+		}
+	}
+	return nil
 }
 
 func VerifyDeletionOfResourcesForRestore(