chore(testing): add skip fail capability to release tests

Until testing.go gets NAT support, the following tests
are expected to fail:

- gatewayPeeringNATTest
- gatewayPeeringOverlapNATTest

Signed-off-by: Pau Capdevila <[email protected]>

Author: pau-hedgehog

pkg/hhfab/release.go

@@ -2540,7 +2540,7 @@ func (testCtx *VPCPeeringTestCtx) gatewayPeeringNATTest(ctx context.Context) (bo
 	// Wait for NAT gateway peering to take effect
 	time.Sleep(15 * time.Second)
 
-	if err := DoVLABTestConnectivity(ctx, testCtx.workDir, testCtx.cacheDir, testCtx.tcOpts); err != nil {
+	if err := DoVLABTestConnectivity(ctx, testCtx.vlabCfg.WorkDir, testCtx.vlabCfg.CacheDir, testCtx.tcOpts); err != nil {
 		return false, reverts, fmt.Errorf("testing NAT gateway peering connectivity: %w", err)
 	}
 
@@ -2706,9 +2706,15 @@ func (testCtx *VPCPeeringTestCtx) gatewayPeeringOverlapNATTest(ctx context.Conte
 	// Wait for network configuration
 	time.Sleep(10 * time.Second)
 
+	// Get SSH connection for server-02
+	serverSSH, err := testCtx.getSSH(ctx, serverToDetach)
+	if err != nil {
+		return false, reverts, fmt.Errorf("getting ssh config for server %s: %w", serverToDetach, err)
+	}
+
 	// Clean up and reconfigure server-02 for the new overlapping VPC
-	if err := execNodeCmd(testCtx.hhfabBin, testCtx.workDir, serverToDetach, "/opt/bin/hhnet cleanup"); err != nil {
-		return false, reverts, fmt.Errorf("cleaning up server %s: %w", serverToDetach, err)
+	if _, stderr, err := serverSSH.Run(ctx, "/opt/bin/hhnet cleanup"); err != nil {
+		return false, reverts, fmt.Errorf("cleaning up server %s: %w: %s", serverToDetach, err, stderr)
 	}
 
 	// Get connection for netconf command
@@ -2723,8 +2729,9 @@ func (testCtx *VPCPeeringTestCtx) gatewayPeeringOverlapNATTest(ctx context.Conte
 		return false, reverts, fmt.Errorf("getting netconf cmd for server %s: %w", serverToDetach, err)
 	}
 
-	if err := execNodeCmd(testCtx.hhfabBin, testCtx.workDir, serverToDetach, fmt.Sprintf("/opt/bin/hhnet %s", netconfCmd)); err != nil {
-		return false, reverts, fmt.Errorf("configuring server %s: %w", serverToDetach, err)
+	cmd := fmt.Sprintf("/opt/bin/hhnet %s", netconfCmd)
+	if _, stderr, err := serverSSH.Run(ctx, cmd); err != nil {
+		return false, reverts, fmt.Errorf("configuring server %s: %w: %s", serverToDetach, err, stderr)
 	}
 
 	// Set up NAT gateway peering between the overlapping VPCs
@@ -2746,7 +2753,7 @@ func (testCtx *VPCPeeringTestCtx) gatewayPeeringOverlapNATTest(ctx context.Conte
 	time.Sleep(15 * time.Second)
 
 	// Test connectivity using the standard test framework
-	if err := DoVLABTestConnectivity(ctx, testCtx.workDir, testCtx.cacheDir, testCtx.tcOpts); err != nil {
+	if err := DoVLABTestConnectivity(ctx, testCtx.vlabCfg.WorkDir, testCtx.vlabCfg.CacheDir, testCtx.tcOpts); err != nil {
 		return false, reverts, fmt.Errorf("testing NAT gateway peering connectivity: %w", err)
 	}
 
@@ -3431,14 +3438,15 @@ type JUnitReport struct {
 }
 
 type JUnitTestSuite struct {
-	XMLName   xml.Name        `xml:"testsuite"`
-	Name      string          `xml:"name,attr"`
-	Tests     int             `xml:"tests,attr"`
-	Failures  int             `xml:"failures,attr"`
-	Skipped   int             `xml:"skipped,attr"`
-	Time      float64         `xml:"time,attr"`
-	TimeHuman time.Duration   `xml:"-"`
-	TestCases []JUnitTestCase `xml:"testcase"`
+	XMLName       xml.Name        `xml:"testsuite"`
+	Name          string          `xml:"name,attr"`
+	Tests         int             `xml:"tests,attr"`
+	Failures      int             `xml:"failures,attr"`
+	Skipped       int             `xml:"skipped,attr"`
+	ExpectedFails int             `xml:"expectedfails,attr"`
+	Time          float64         `xml:"time,attr"`
+	TimeHuman     time.Duration   `xml:"-"`
+	TestCases     []JUnitTestCase `xml:"testcase"`
 }
 
 type SkipFlags struct {
@@ -3504,14 +3512,21 @@ func (sf *SkipFlags) PrettyPrint() string {
 }
 
 type JUnitTestCase struct {
-	XMLName   xml.Name  `xml:"testcase"`
-	ClassName string    `xml:"classname,attr"`
-	Name      string    `xml:"name,attr"`
-	Time      float64   `xml:"time,attr"`
-	Failure   *Failure  `xml:"failure,omitempty"`
-	Skipped   *Skipped  `xml:"skipped,omitempty"`
-	F         TestFunc  `xml:"-"` // function to run
-	SkipFlags SkipFlags `xml:"-"` // flags to determine whether to skip the test
+	XMLName       xml.Name      `xml:"testcase"`
+	ClassName     string        `xml:"classname,attr"`
+	Name          string        `xml:"name,attr"`
+	Time          float64       `xml:"time,attr"`
+	Failure       *Failure      `xml:"failure,omitempty"`
+	Skipped       *Skipped      `xml:"skipped,omitempty"`
+	ExpectedFail  *ExpectedFail `xml:"expectedfail,omitempty"`
+	F             TestFunc      `xml:"-"` // function to run
+	SkipFlags     SkipFlags     `xml:"-"` // flags to determine whether to skip the test
+	ExpectFailure bool          `xml:"-"`
+}
+
+type ExpectedFail struct {
+	XMLName xml.Name `xml:"expectedfail"`
+	Message string   `xml:"message,attr,omitempty"`
 }
 
 type Failure struct {
@@ -3533,21 +3548,25 @@ func printTestSuite(ts *JUnitTestSuite) {
 }
 
 func printSuiteResults(ts *JUnitTestSuite) {
-	var numFailed, numSkipped, numPassed int
+	var numFailed, numSkipped, numPassed, numXFail int
 	slog.Info("Test suite results", "suite", ts.Name)
 	for _, test := range ts.TestCases {
-		if test.Skipped != nil { //nolint:gocritic
+		switch {
+		case test.Skipped != nil:
 			slog.Warn("SKIP", "test", test.Name, "reason", test.Skipped.Message)
 			numSkipped++
-		} else if test.Failure != nil {
+		case test.ExpectedFail != nil:
+			slog.Info("XFAIL", "test", test.Name, "reason", test.ExpectedFail.Message)
+			numXFail++
+		case test.Failure != nil:
 			slog.Error("FAIL", "test", test.Name, "error", strings.ReplaceAll(test.Failure.Message, "\n", "; "))
 			numFailed++
-		} else {
+		default:
 			slog.Info("PASS", "test", test.Name)
 			numPassed++
 		}
 	}
-	slog.Info("Test suite summary", "tests", len(ts.TestCases), "passed", numPassed, "skipped", numSkipped, "failed", numFailed, "duration", ts.TimeHuman)
+	slog.Info("Test suite summary", "tests", len(ts.TestCases), "passed", numPassed, "skipped", numSkipped, "failed", numFailed, "xfail", numXFail, "duration", ts.TimeHuman)
 }
 
 func pauseOnFail() {
@@ -3610,7 +3629,8 @@ func doRunTests(ctx context.Context, testCtx *VPCPeeringTestCtx, ts *JUnitTestSu
 		// - we then apply reverts in reverse order, and if any of them fails, we mark the test as failed, and pause (potentially a second time) if configured to do so.
 		//   we also stop applying reverts at the first failure
 		// - finally, if we get to the end without any errors, we log the test as passed
-		if skip {
+		switch {
+		case skip:
 			var skipMsg string
 			if err != nil {
 				skipMsg = err.Error()
@@ -3624,8 +3644,14 @@ func doRunTests(ctx context.Context, testCtx *VPCPeeringTestCtx, ts *JUnitTestSu
 			}
 			ts.Skipped++
 			slog.Warn("SKIP", "test", test.Name, "reason", skipMsg)
-		}
-		if err != nil {
+		case test.ExpectFailure && err != nil:
+			ts.TestCases[i].ExpectedFail = &ExpectedFail{
+				Message: err.Error(),
+			}
+			ts.ExpectedFails++
+			slog.Info("XFAIL", "test", test.Name, "reason", err.Error())
+			err = nil
+		case err != nil:
 			ts.TestCases[i].Failure = &Failure{
 				Message: err.Error(),
 			}
@@ -3705,10 +3731,17 @@ func failAllTests(suite *JUnitTestSuite, err error) *JUnitTestSuite {
 		if suite.TestCases[i].Skipped != nil {
 			continue
 		}
-		suite.TestCases[i].Failure = &Failure{
-			Message: err.Error(),
+		if suite.TestCases[i].ExpectFailure {
+			suite.TestCases[i].ExpectedFail = &ExpectedFail{
+				Message: err.Error(),
+			}
+			suite.ExpectedFails++
+		} else {
+			suite.TestCases[i].Failure = &Failure{
+				Message: err.Error(),
+			}
+			suite.Failures++
 		}
-		suite.Failures++
 	}
 
 	return suite
@@ -4057,15 +4090,17 @@ func makeVpcPeeringsBasicSuite(testCtx *VPCPeeringTestCtx) *JUnitTestSuite {
 			},
 		},
 		{
-			Name: "Gateway Peering with NAT",
-			F:    testCtx.gatewayPeeringNATTest,
+			Name:          "Gateway Peering with NAT",
+			F:             testCtx.gatewayPeeringNATTest,
+			ExpectFailure: true,
 			SkipFlags: SkipFlags{
 				NoGateway: true,
 			},
 		},
 		{
-			Name: "Gateway Peering Overlapping Subnets with NAT",
-			F:    testCtx.gatewayPeeringOverlapNATTest,
+			Name:          "Gateway Peering Overlapping Subnets with NAT",
+			F:             testCtx.gatewayPeeringOverlapNATTest,
+			ExpectFailure: true,
 			SkipFlags: SkipFlags{
 				NoGateway: true,
 			},