Skip to content

Commit

Permalink
add MemberDowngradeUpgrade failpoint
Browse files Browse the repository at this point in the history 
Signed-off-by: Siyuan Zhang <[email protected]>
  • Loading branch information
@siyuanfoundation
siyuanfoundation committed Jan 13, 2025
1 parent ebb2b06 commit 4c9d908
Show file tree
Hide file tree
Showing 5 changed files with 198 additions and 98 deletions.
62 changes: 4 additions & 58 deletions tests/e2e/cluster_downgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,13 @@ package e2e

import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"

"github.com/coreos/go-semver/semver"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"

"go.etcd.io/etcd/api/v3/version"
"go.etcd.io/etcd/client/pkg/v3/fileutil"
Expand Down Expand Up @@ -86,7 +83,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
var snapshotCount uint64 = 10
epc := newCluster(t, clusterSize, snapshotCount)
for i := 0; i < len(epc.Procs); i++ {
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
Cluster: currentVersionStr,
Server: version.Version,
Storage: currentVersionStr,
Expand Down Expand Up @@ -117,7 +114,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {

t.Log("Downgrade enabled, validating if cluster is ready for downgrade")
for i := 0; i < len(epc.Procs); i++ {
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
Cluster: lastClusterVersionStr,
Server: version.Version,
Storage: lastClusterVersionStr,
Expand All @@ -136,7 +133,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
t.Log("All members downgraded, validating downgrade")
e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")
for i := 0; i < len(epc.Procs); i++ {
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
Cluster: lastClusterVersionStr,
Server: lastVersionStr,
})
Expand Down Expand Up @@ -176,7 +173,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {

t.Log("All members upgraded, validating upgrade")
for i := 0; i < len(epc.Procs); i++ {
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
Cluster: currentVersionStr,
Server: version.Version,
Storage: currentVersionStr,
Expand Down Expand Up @@ -227,27 +224,6 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) {
require.NoError(t, err)
}

func validateVersion(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, expect version.Versions) {
testutils.ExecuteWithTimeout(t, 30*time.Second, func() {
for {
result, err := getMemberVersionByCurl(cfg, member)
if err != nil {
cfg.Logger.Warn("failed to get member version and retrying", zap.Error(err), zap.String("member", member.Config().Name))
time.Sleep(time.Second)
continue
}
cfg.Logger.Info("Comparing versions", zap.String("member", member.Config().Name), zap.Any("got", result), zap.Any("want", expect))
if err := compareMemberVersion(expect, result); err != nil {
cfg.Logger.Warn("Versions didn't match retrying", zap.Error(err), zap.String("member", member.Config().Name))
time.Sleep(time.Second)
continue
}
cfg.Logger.Info("Versions match", zap.String("member", member.Config().Name))
break
}
})
}

func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
Expand All @@ -269,36 +245,6 @@ func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess {
return nil
}

func compareMemberVersion(expect version.Versions, target version.Versions) error {
if expect.Server != "" && expect.Server != target.Server {
return fmt.Errorf("expect etcdserver version %v, but got %v", expect.Server, target.Server)
}

if expect.Cluster != "" && expect.Cluster != target.Cluster {
return fmt.Errorf("expect etcdcluster version %v, but got %v", expect.Cluster, target.Cluster)
}

if expect.Storage != "" && expect.Storage != target.Storage {
return fmt.Errorf("expect storage version %v, but got %v", expect.Storage, target.Storage)
}
return nil
}

func getMemberVersionByCurl(cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) (version.Versions, error) {
args := e2e.CURLPrefixArgsCluster(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"})
lines, err := e2e.RunUtilCompletion(args, nil)
if err != nil {
return version.Versions{}, err
}

data := strings.Join(lines, "\n")
result := version.Versions{}
if err := json.Unmarshal([]byte(data), &result); err != nil {
return version.Versions{}, fmt.Errorf("failed to unmarshal (%v): %w", data, err)
}
return result, nil
}

func generateSnapshot(t *testing.T, snapshotCount uint64, cc *e2e.EtcdctlV3) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down
6 changes: 2 additions & 4 deletions tests/framework/e2e/curl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,8 @@ func CURLPut(clus *EtcdProcessCluster, req CURLReq) error {
}

func CURLGet(clus *EtcdProcessCluster, req CURLReq) error {
ctx, cancel := context.WithTimeout(context.Background(), req.timeoutDuration())
defer cancel()

return SpawnWithExpectsContext(ctx, CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "GET", req), nil, req.Expected)
member := clus.Procs[rand.Intn(clus.Cfg.ClusterSize)]
return CURLGetFromMember(clus, member, req)
}

func CURLGetFromMember(clus *EtcdProcessCluster, member EtcdProcess, req CURLReq) error {
Expand Down
62 changes: 62 additions & 0 deletions tests/framework/e2e/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@ import (
"testing"
"time"

"github.com/coreos/go-semver/semver"
"go.uber.org/zap"

"go.etcd.io/etcd/api/v3/version"
"go.etcd.io/etcd/client/pkg/v3/testutil"
"go.etcd.io/etcd/pkg/v3/expect"
"go.etcd.io/etcd/tests/v3/framework/testutils"
)

func WaitReadyExpectProc(ctx context.Context, exproc *expect.ExpectProcess, readyStrs []string) error {
Expand Down Expand Up @@ -177,3 +182,60 @@ func mergeEnvVariables(envVars map[string]string) []string {

return env
}

func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdProcess, expect version.Versions) {
testutils.ExecuteWithTimeout(t, 30*time.Second, func() {
for {
result, err := getMemberVersionByCurl(cfg, member)

Check failure

Code scanning / CodeQL

Clear-text logging of sensitive information High test

Sensitive data returned by an access to Password
Loading
flows to a logging call.
if err != nil {
cfg.Logger.Warn("failed to get member version and retrying", zap.Error(err), zap.String("member", member.Config().Name))
time.Sleep(time.Second)
continue
}
cfg.Logger.Info("Comparing versions", zap.String("member", member.Config().Name), zap.Any("got", result), zap.Any("want", expect))
if err := compareMemberVersion(expect, result); err != nil {
cfg.Logger.Warn("Versions didn't match retrying", zap.Error(err), zap.String("member", member.Config().Name))
time.Sleep(time.Second)
continue
}
cfg.Logger.Info("Versions match", zap.String("member", member.Config().Name))
break
}
})
}

func majorMinorVersionsEqual(v1, v2 string) bool {
ver1 := semver.New(v1)
ver2 := semver.New(v2)
return ver1.Major == ver2.Major && ver1.Minor == ver2.Minor
}

func compareMemberVersion(expect version.Versions, target version.Versions) error {
if expect.Server != "" && !majorMinorVersionsEqual(expect.Server, target.Server) {
return fmt.Errorf("expect etcdserver version %v, but got %v", expect.Server, target.Server)
}

if expect.Cluster != "" && !majorMinorVersionsEqual(expect.Cluster, target.Cluster) {
return fmt.Errorf("expect etcdcluster version %v, but got %v", expect.Cluster, target.Cluster)
}

if expect.Storage != "" && !majorMinorVersionsEqual(expect.Storage, target.Storage) {
return fmt.Errorf("expect storage version %v, but got %v", expect.Storage, target.Storage)
}
return nil
}

func getMemberVersionByCurl(cfg *EtcdProcessClusterConfig, member EtcdProcess) (version.Versions, error) {
args := CURLPrefixArgsCluster(cfg, member, "GET", CURLReq{Endpoint: "/version"})
lines, err := RunUtilCompletion(args, nil)
if err != nil {
return version.Versions{}, err
}

data := strings.Join(lines, "\n")
result := version.Versions{}
if err := json.Unmarshal([]byte(data), &result); err != nil {
return version.Versions{}, fmt.Errorf("failed to unmarshal (%v): %w", data, err)
}
return result, nil
}
Loading

0 comments on commit 4c9d908

Please sign in to comment.