test: refactor suite to use suite dependencies and concurrent setup / teardown

Author: lizardruss

Justfile

@@ -110,7 +110,7 @@ setup-csi-volume-snapshots:
   (run-e2e label-filter image) \
   (teardown label-filter)
 
-@run-e2e label-filter="core" image="ghcr.io/loft-sh/vcluster:dev-next" teardown="false":
+@run-e2e label-filter="core" image="ghcr.io/loft-sh/vcluster:dev-next" teardown="true":
   ginkgo -timeout=0 -v --label-filter="{{label-filter}}" ./e2e-next -- --vcluster-image="{{image}}" --teardown={{teardown}}
 
 @iterate-e2e label-filter="core" image="ghcr.io/loft-sh/vcluster:dev-next": \

e2e-next/clusters/clusters.go

@@ -0,0 +1,100 @@
+package clusters
+
+import (
+	_ "embed"
+
+	"os"
+	"path/filepath"
+
+	"github.com/loft-sh/e2e-framework/pkg/provider/kind"
+	providervcluster "github.com/loft-sh/e2e-framework/pkg/provider/vcluster"
+	"github.com/loft-sh/e2e-framework/pkg/setup/cluster"
+	"github.com/loft-sh/e2e-framework/pkg/setup/vcluster"
+	"github.com/loft-sh/vcluster/e2e-next/constants"
+	"github.com/loft-sh/vcluster/e2e-next/setup/template"
+	"sigs.k8s.io/e2e-framework/support"
+)
+
+var (
+	HostCluster = cluster.Define(
+		cluster.WithName(constants.GetHostClusterName()),
+		cluster.WithProvider(kind.NewProvider()),
+		cluster.WithConfigFile("e2e-kind.config.yaml"),
+	)
+)
+
+var (
+	//go:embed vcluster-default.yaml
+	DefaultVClusterYAMLTemplate string
+	DefaultVClusterVars         map[string]interface{} = map[string]interface{}{
+		"Repository": constants.GetRepository(),
+		"Tag":        constants.GetTag(),
+	}
+
+	DefaultVClusterYAML, DefaultVClusterYAMLCleanup = template.MustRender(DefaultVClusterYAMLTemplate, DefaultVClusterVars)
+	DefaultVClusterOptions                          = []support.ClusterOpts{
+		providervcluster.WithPath(filepath.Join(os.Getenv("GOBIN"), "vcluster")),
+		providervcluster.WithLocalChartDir("../chart"),
+		providervcluster.WithUpgrade(true),
+		providervcluster.WithBackgroundProxyImage(constants.GetVclusterImage()),
+	}
+)
+var (
+	K8sDefaultEndpointVClusterName = "k8s-default-endpoint-test"
+	K8sDefaultEndpointVCluster     = vcluster.Define(
+		vcluster.WithName(K8sDefaultEndpointVClusterName),
+		vcluster.WithVClusterYAML(DefaultVClusterYAML),
+		vcluster.WithOptions(
+			DefaultVClusterOptions...,
+		),
+		vcluster.WithDependencies(HostCluster),
+	)
+)
+
+var (
+	NodesVClusterName = "nodes-test-vcluster"
+	NodesVCluster     = vcluster.Define(
+		vcluster.WithName(NodesVClusterName),
+		vcluster.WithVClusterYAML(DefaultVClusterYAML),
+		vcluster.WithOptions(
+			DefaultVClusterOptions...,
+		),
+		vcluster.WithDependencies(HostCluster),
+	)
+)
+
+var (
+	//go:embed vcluster-test-helm.yaml
+	HelmChartsVClusterYAMLTemplate                        string
+	HelmChartsVClusterYAML, HelmChartsVClusterYAMLCleanup = template.MustRender(
+		HelmChartsVClusterYAMLTemplate,
+		DefaultVClusterVars,
+	)
+	HelmChartsVClusterName = "helm-charts-test-vcluster"
+	HelmChartsVCluster     = vcluster.Define(
+		vcluster.WithName(HelmChartsVClusterName),
+		vcluster.WithVClusterYAML(HelmChartsVClusterYAML),
+		vcluster.WithOptions(
+			DefaultVClusterOptions...,
+		),
+		vcluster.WithDependencies(HostCluster),
+	)
+)
+
+var (
+	//go:embed vcluster-init-manifest.yaml
+	InitManifestsVClusterTemplate                               string
+	InitManifestsVClusterName                                   = "init-manifests-test-vcluster"
+	InitManifestsVClusterYAML, InitManifestsVClusterYAMLCleanup = template.MustRender(
+		InitManifestsVClusterTemplate,
+		DefaultVClusterVars,
+	)
+	InitManifestsVCluster = vcluster.Define(
+		vcluster.WithName(InitManifestsVClusterName),
+		vcluster.WithVClusterYAML(InitManifestsVClusterYAML),
+		vcluster.WithOptions(
+			DefaultVClusterOptions...,
+		),
+		vcluster.WithDependencies(HostCluster),
+	)
+)

e2e-next/constants/defaultvcluster.yaml e2e-next/clusters/vcluster-default.yamle2e-next/constants/defaultvcluster.yaml renamed to e2e-next/clusters/vcluster-default.yaml

@@ -2,8 +2,8 @@ controlPlane:
   statefulSet:
     image:
       registry: ""
-      repository: {{.Repository}}
-      tag: {{.Tag}}
+      repository: {{ .Repository }}
+      tag: {{ .Tag }}
 experimental:
   deploy:
     vcluster:
@@ -20,4 +20,4 @@ experimental:
         metadata:
           name: test-configmap-2
         data:
-          foo: {{ .Release.Name }}
+          foo: {{ printf "{{ .Release.Name }}" }}

…/test_deploy/vcluster-init-manifest.yaml …ext/clusters/vcluster-init-manifest.yamle2e-next/test_deploy/vcluster-init-manifest.yaml renamed to e2e-next/clusters/vcluster-init-manifest.yaml e2e-next/test_deploy/vcluster-init-manifest.yaml renamed to e2e-next/clusters/vcluster-init-manifest.yaml

@@ -7,7 +7,10 @@ import (
 	"testing"
 
 	"github.com/loft-sh/e2e-framework/pkg/e2e"
-	cluster "github.com/loft-sh/e2e-framework/pkg/setup/cluster"
+	"github.com/loft-sh/e2e-framework/pkg/setup"
+	"github.com/loft-sh/e2e-framework/pkg/setup/cluster"
+	"github.com/loft-sh/e2e-framework/pkg/setup/suite"
+	"github.com/loft-sh/vcluster/e2e-next/clusters"
 	"github.com/peterbourgon/ff/v3"
 
 	"github.com/loft-sh/vcluster/e2e-next/constants"
@@ -16,7 +19,9 @@ import (
 	"github.com/onsi/gomega/format"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
-	"sigs.k8s.io/e2e-framework/support/kind"
+
+	// Initialize framework
+	_ "github.com/loft-sh/vcluster/e2e-next/init"
 
 	// Import tests
 	_ "github.com/loft-sh/vcluster/e2e-next/test_core/sync"
@@ -54,56 +59,89 @@ func handleFlags() {
 	e2e.SetTeardownOnly(teardownOnly)
 }
 
-// This must be called before any ginkgo DSL evaluation
-var _ = AddTreeConstructionNodeArgsTransformer(e2e.ContextualNodeTransformer)
-
 func TestMain(m *testing.M) {
 	handleFlags()
 	os.Exit(m.Run())
 }
 
 func TestRunE2ETests(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "vCluster E2E Suite", AroundNode(e2e.ContextualAroundNode))
-}
-
-var _ = BeforeSuite(func(ctx context.Context) context.Context {
-	var err error
-
 	// Disable Ginkgo's truncating of long lines'
 	format.MaxLength = 0
+	config, _ := GinkgoConfiguration()
 
-	By("Creating kind cluster " + clusterName)
-	clusterOptions := []cluster.Options{
-		cluster.WithName(constants.GetHostClusterName()),
-		cluster.WithProvider(kind.NewProvider()),
-	}
-	if constants.GetHostClusterName() == "kind-cluster" {
-		clusterOptions = append(clusterOptions, cluster.WithConfigFile("e2e-kind.config.yaml"))
-	}
-
-	ctx, err = cluster.Create(clusterOptions...)(ctx)
-	Expect(err).NotTo(HaveOccurred())
-
-	By("Setting up controller runtime client for " + clusterName)
-	ctx, err = cluster.SetupControllerRuntimeClient(cluster.WithCluster(clusterName))(ctx)
-	Expect(err).NotTo(HaveOccurred())
-
-	ctx, err = cluster.SetupKubeClient(clusterName)(ctx)
-	Expect(err).NotTo(HaveOccurred())
-
-	By("Setting current cluster to " + clusterName)
-	ctx, err = cluster.UseCluster(clusterName)(ctx)
-	Expect(err).NotTo(HaveOccurred())
-
-	By("Loading image to kind cluster...")
-	ctx, err = cluster.LoadImage(clusterName, vclusterImage)(ctx)
-	Expect(err).NotTo(HaveOccurred())
+	RegisterFailHandler(Fail)
+	RunSpecs(
+		t,
+		"vCluster E2E Suite",
+		AroundNode(suite.PreviewSpecsAroundNode(config)),
+		AroundNode(e2e.ContextualAroundNode),
+	)
+}
 
-	return ctx
-})
+var _ = SynchronizedBeforeSuite(
+	func(ctx context.Context) (context.Context, []byte) {
+		var err error
+
+		// Clean up vcluster yaml
+		DeferCleanup(clusters.DefaultVClusterYAMLCleanup)
+		DeferCleanup(clusters.HelmChartsVClusterYAMLCleanup)
+		DeferCleanup(clusters.InitManifestsVClusterYAMLCleanup)
+
+		ctx, err = setup.All(
+			clusters.HostCluster.Setup,
+			func(ctx context.Context) (context.Context, error) {
+				var err error
+				By("Loading image to kind cluster...", func() {
+					ctx, err = cluster.LoadImage(clusterName, vclusterImage)(ctx)
+					Expect(err).NotTo(HaveOccurred())
+				})
+				return ctx, err
+			},
+			func(ctx context.Context) (context.Context, error) {
+				var err error
+				By("Creating all virtual clusters...", func() {
+					ctx, err = setup.AllConcurrent(
+						clusters.K8sDefaultEndpointVCluster.Setup,
+						clusters.NodesVCluster.Setup,
+						clusters.HelmChartsVCluster.Setup,
+						clusters.InitManifestsVCluster.Setup,
+					)(ctx)
+					Expect(err).NotTo(HaveOccurred())
+				})
+				return ctx, err
+			},
+		)(ctx)
+		Expect(err).NotTo(HaveOccurred())
+
+		data, err := cluster.ExportAll(ctx)
+		Expect(err).NotTo(HaveOccurred())
+
+		return ctx, data
+	},
+	func(ctx context.Context, data []byte) context.Context {
+		var err error
+
+		ctx, err = cluster.ImportAll(ctx, data)
+		Expect(err).NotTo(HaveOccurred())
+
+		return ctx
+	},
+)
 
-var _ = AfterSuite(func(ctx context.Context) {
-	_, err := cluster.Destroy(constants.GetHostClusterName())(ctx)
-	Expect(err).NotTo(HaveOccurred())
-})
+var _ = SynchronizedAfterSuite(
+	func(ctx context.Context) {
+	},
+	func(ctx context.Context) {
+		_, err := setup.All(
+			// Sometimes namespace finalizers take a while... so let's just delete the host cluster.
+			//setup.AllConcurrent(
+			//	clusters.K8sDefaultEndpointVCluster.Teardown,
+			//	clusters.NodesVCluster.Teardown,
+			//	clusters.HelmChartsVCluster.Teardown,
+			//	clusters.InitManifestsVCluster.Teardown,
+			//),
+			clusters.HostCluster.Teardown,
+		)(ctx)
+		Expect(err).NotTo(HaveOccurred())
+	},
+)

…next/test_deploy/vcluster-test-helm.yaml e2e-next/clusters/vcluster-test-helm.yamle2e-next/test_deploy/vcluster-test-helm.yaml renamed to e2e-next/clusters/vcluster-test-helm.yaml e2e-next/test_deploy/vcluster-test-helm.yaml renamed to e2e-next/clusters/vcluster-test-helm.yaml

@@ -0,0 +1,11 @@
+package init
+
+import (
+	"github.com/loft-sh/e2e-framework/pkg/e2e"
+	"github.com/loft-sh/e2e-framework/pkg/setup/suite"
+	. "github.com/onsi/ginkgo/v2"
+)
+
+// This must be called before any ginkgo DSL evaluation
+var _ = AddTreeConstructionNodeArgsTransformer(suite.NodeTransformer)
+var _ = AddTreeConstructionNodeArgsTransformer(e2e.ContextualNodeTransformer)

e2e-next/constants/vcluster.go

@@ -0,0 +1,37 @@
+package template
+
+import (
+	"os"
+	gotemplate "text/template"
+)
+
+func MustRender(template string, data interface{}) (string, func() error) {
+	tempDir, cleanUp, err := Render(template, data)
+	if err != nil {
+		panic(err)
+	}
+	return tempDir, cleanUp
+}
+
+func Render(template string, data interface{}) (string, func() error, error) {
+	tmpFile, err := os.CreateTemp("", "vcluster-*.yaml")
+	if err != nil {
+		return "", nil, err
+	}
+	defer func(tmpFile *os.File) {
+		_ = tmpFile.Close()
+	}(tmpFile)
+
+	parsedTemplate, err := gotemplate.New("template").Parse(template)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if err := parsedTemplate.Execute(tmpFile, data); err != nil {
+		return "", nil, err
+	}
+
+	return tmpFile.Name(), func() error {
+		return os.Remove(tmpFile.Name())
+	}, nil
+}