e2e: add upgrade test for clusters set up by promoted members

Signed-off-by: Wei Fu <[email protected]>

Author: fuweid

tests/e2e/etcd_release_upgrade_test.go

@@ -16,17 +16,21 @@ package e2e
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/pkg/v3/expect"
+	"go.etcd.io/etcd/tests/v3/framework/config"
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
+	"go.etcd.io/etcd/tests/v3/framework/testutils"
 )
 
 // TestReleaseUpgrade ensures that changes to master branch does not affect
@@ -165,3 +169,113 @@ func TestReleaseUpgradeWithRestart(t *testing.T) {
 
 	require.NoError(t, ctlV3Get(cx, []string{kvs[0].key}, []kv{kvs[0]}...))
 }
+
+func TestClusterUpgradeAfterPromotingMembers(t *testing.T) {
+	if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
+		t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease)
+	}
+
+	e2e.BeforeTest(t)
+
+	currentVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
+	require.NoErrorf(t, err, "failed to get version from binary")
+
+	lastClusterVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease)
+	require.NoErrorf(t, err, "failed to get version from last release binary")
+
+	clusterSize := 3
+
+	epc := createNewClusterByPromotingMembers(t, e2e.LastVersion, clusterSize)
+	defer func() {
+		require.NoError(t, epc.Close())
+	}()
+
+	err = e2e.DowngradeUpgradeMembers(t, nil, epc, 3, false, lastClusterVersion, currentVersion)
+	require.NoError(t, err)
+
+	t.Log("Checking all member status after upgrading")
+	mresp, merr := epc.Etcdctl().MemberList(t.Context(), true)
+	require.NoError(t, merr)
+	require.Len(t, mresp.Members, clusterSize)
+	for _, m := range mresp.Members {
+		require.Falsef(t, m.IsLearner, "%s should not be learner", m.Name)
+	}
+}
+
+func createNewClusterByPromotingMembers(t *testing.T, clusterVersion e2e.ClusterVersion, clusterSize int) *e2e.EtcdProcessCluster {
+	require.Truef(t, clusterSize >= 1, "clusterSize >= 1")
+
+	var version *semver.Version
+	var err error
+
+	switch clusterVersion {
+	case e2e.CurrentVersion:
+		version, err = e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
+		require.NoErrorf(t, err, "failed to get version from binary")
+	case e2e.LastVersion:
+		if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
+			t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease)
+		}
+
+		version, err = e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease)
+		require.NoErrorf(t, err, "failed to get version from last release binary")
+	default:
+		t.Fatalf("unexpected cluster version: %v", clusterVersion)
+	}
+
+	t.Logf("Creating new etcd cluster - version: %v, clusterSize: %v", version, clusterSize)
+
+	t.Log("Creating first node")
+	epc, err := e2e.NewEtcdProcessCluster(t.Context(), t,
+		e2e.WithVersion(clusterVersion),
+		e2e.WithClusterSize(1),
+		e2e.WithSnapshotCount(10),
+	)
+	require.NoErrorf(t, err, "failed to start first etcd process")
+	defer func() {
+		if t.Failed() {
+			epc.Close()
+		}
+	}()
+
+	for i := 1; i < clusterSize; i++ {
+		var nodeID uint64
+		var aerr error
+
+		// NOTE: New promoted member needs time to get connected.
+		t.Logf("[%d] Adding new node as learner", i)
+		testutils.ExecuteWithTimeout(t, 1*time.Minute, func() {
+			for {
+				nodeID, aerr = epc.StartNewProc(t.Context(), nil, t, true)
+				if aerr != nil {
+					if strings.Contains(aerr.Error(), "etcdserver: unhealthy cluster") {
+						time.Sleep(1 * time.Second)
+						continue
+					}
+				}
+				break
+			}
+		})
+		require.NoError(t, aerr)
+
+		t.Logf("[%d] Promoting node(%d)", i, nodeID)
+		etcdctl := epc.Procs[0].Etcdctl()
+		_, err = etcdctl.MemberPromote(t.Context(), nodeID)
+		require.NoError(t, err)
+	}
+
+	t.Log("Checking all member status")
+	mresp, merr := epc.Etcdctl().MemberList(t.Context(), true)
+	require.NoError(t, merr)
+	require.Len(t, mresp.Members, clusterSize)
+	for _, m := range mresp.Members {
+		require.Falsef(t, m.IsLearner, "%s should not be learner", m.Name)
+	}
+
+	t.Logf("Adding 10 key/value to trigger snapshot")
+	for i := 0; i < 10; i++ {
+		err = epc.Etcdctl().Put(t.Context(), "foo", "bar", config.PutOptions{})
+		require.NoError(t, err)
+	}
+	return epc
+}