e2e: surface checksum mismatch for verification and deletion

Author: skoya76

test/e2e/delete_incremental_backup_test.go

@@ -1,9 +1,13 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"time"
 
+	"path/filepath"
+	"strconv"
+
 	finv1 "github.com/cybozu-go/fin/api/v1"
 	"github.com/cybozu-go/fin/internal/controller"
 	"github.com/cybozu-go/fin/test/utils"
@@ -118,4 +122,34 @@ func deleteIncrementalBackupTestSuite() {
 			return nil
 		}, "10s", "1s").WithContext(ctx).Should(Succeed())
 	})
+
+	It("should fail deletion job when checksum file is corrupted", func(ctx SpecContext) {
+		// Arrange
+		pod := CreatePodForBlockPVC(ctx, k8sClient, pvc)
+		_ = WriteRandomDataToPVC(ctx, pod, devicePathInPodForPVC, 1024)
+
+		fullBackup := CreateBackup(ctx, ctrlClient, rookNamespace, pvc, nodes[0])
+
+		incremental, err := NewFinBackup(rookNamespace, utils.GetUniqueName("test-finbackup-"), pvc, nodes[0])
+		incremental.Annotations = map[string]string{controller.AnnotationSkipVerify: "true"}
+		Expect(err).NotTo(HaveOccurred())
+		Expect(CreateFinBackup(ctx, ctrlClient, incremental)).NotTo(HaveOccurred())
+		Expect(WaitForFinBackupStoredToNode(ctx, ctrlClient, incremental, 2*time.Minute)).NotTo(HaveOccurred())
+		Expect(ctrlClient.Get(ctx, client.ObjectKeyFromObject(incremental), incremental)).NotTo(HaveOccurred())
+		ExpectDiffChecksumExists(nodes[0], incremental, pvc)
+
+		diffChecksumPath := filepath.Join("/fin", pvc.Namespace, pvc.Name, "diff", strconv.Itoa(*incremental.Status.SnapID), "part-0.csum")
+		CorruptFileOnNode(nodes[0], diffChecksumPath)
+
+		// Act
+		Expect(DeleteFinBackup(ctx, ctrlClient, incremental)).NotTo(HaveOccurred())
+
+		// Assert
+		Expect(WaitForFinBackupChecksumMismatch(ctx, ctrlClient, incremental, 2*time.Minute)).NotTo(HaveOccurred())
+
+		// Cleanup
+		ForceDeleteFinBackup(context.Background(), ctrlClient, incremental)
+		ForceDeleteFinBackup(context.Background(), ctrlClient, fullBackup)
+		_ = DeletePod(ctx, k8sClient, pod)
+	})
 }

test/e2e/verification_test.go

@@ -1,7 +1,9 @@
 package e2e
 
 import (
+	"context"
 	"path"
+	"strconv"
 	"time"
 
 	"github.com/cybozu-go/fin/test/utils"
@@ -289,4 +291,34 @@ func verificationTestSuite() {
 			Expect(err).NotTo(HaveOccurred())
 		},
 	)
+
+	It("should fail verification when diff checksum file is corrupted", func(ctx SpecContext) {
+		// Arrange
+		fullBackup := CreateBackup(ctx, ctrlClient, rookNamespace, pvc, nodes[0])
+
+		By("modifying data to generate diff for incremental backup")
+		pod := CreatePodForFilesystemPVC(ctx, k8sClient, pvc)
+		defer func() {
+			_ = DeletePod(context.Background(), k8sClient, pod)
+		}()
+		_ = WriteRandomDataToPVC(ctx, pod, path.Join(mountPathInPodForFSPVC, "test2"), dataSize)
+
+		By("creating an incremental backup")
+		incrementalBackup, err := NewFinBackup(ns.Name, utils.GetUniqueName("test-finbackup-"), pvc, nodes[0])
+		Expect(err).NotTo(HaveOccurred())
+		Expect(CreateFinBackup(ctx, ctrlClient, incrementalBackup)).NotTo(HaveOccurred())
+		Expect(WaitForFinBackupStoredToNode(ctx, ctrlClient, incrementalBackup, 2*time.Minute)).NotTo(HaveOccurred())
+		Expect(ctrlClient.Get(ctx, client.ObjectKeyFromObject(incrementalBackup), incrementalBackup)).NotTo(HaveOccurred())
+		ExpectDiffChecksumExists(nodes[0], incrementalBackup, pvc)
+
+		diffChecksumPath := path.Join("/fin", pvc.Namespace, pvc.Name, "diff", strconv.Itoa(*incrementalBackup.Status.SnapID), "part-0.csum")
+		CorruptFileOnNode(nodes[0], diffChecksumPath)
+
+		// Act + Assert
+		Expect(WaitForFinBackupChecksumMismatch(ctx, ctrlClient, incrementalBackup, 2*time.Minute)).NotTo(HaveOccurred())
+
+		// Cleanup
+		ForceDeleteFinBackup(context.Background(), ctrlClient, incrementalBackup)
+		ForceDeleteFinBackup(context.Background(), ctrlClient, fullBackup)
+	})
 }