server: add fields to migrate from 3.5 to 3.4

etcdutl/etcdutl/migrate_command.go

@@ -87,8 +87,8 @@ func (o *migrateOptions) Config() (*migrateConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse target version: %v", err)
 	}
-	if c.targetVersion.LessThan(version.V3_5) {
-		return nil, fmt.Errorf(`target version %q not supported. Minimal "3.5"`, storageVersionToString(c.targetVersion))
+	if c.targetVersion.LessThan(version.V3_4) {
+		return nil, fmt.Errorf(`target version %q not supported. Minimal "3.4"`, storageVersionToString(c.targetVersion))
 	}
 
 	dbPath := datadir.ToBackendFileName(o.dataDir)

scripts/test.sh

@@ -564,44 +564,82 @@ function dep_pass {
 
 function release_pass {
   rm -f ./bin/etcd-last-release
+  rm -f ./bin/etcd-before-last-release
+  mkdir -p ./bin
 
-  # Work out the previous release based on the version reported by etcd binary
+  # Work out two previous releases based on the version reported by etcd binary
   binary_version=$(./bin/etcd --version | grep --only-matching --perl-regexp '(?<=etcd Version: )\d+\.\d+')
   binary_major=$(echo "${binary_version}" | cut -d '.' -f 1)
   binary_minor=$(echo "${binary_version}" | cut -d '.' -f 2)
+  previous_major=$binary_major
   previous_minor=$((binary_minor - 1))
+  before_previous_major=$binary_major
+  before_previous_minor=$((binary_minor - 2))
 
   # Handle the edge case where we go to a new major version
   # When this happens we obtain latest minor release of previous major
   if [ "${binary_minor}" -eq 0 ]; then
-    binary_major=$((binary_major - 1))
+    previous_major=$((binary_major - 1))
+    before_previous_major=$previous_minor
     previous_minor=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \
     | grep --only-matching --perl-regexp "(?<=v)${binary_major}.\d.[\d]+?(?=[\^])" \
     | sort --numeric-sort --key 1.3 | tail -1 | cut -d '.' -f 2)
+    before_previous_minor=$((previous_minor - 1))
+  fi
+
+  # Handle the edge case when only 'before previous' should be latest minor release of previous major
+  if [ "${binary_minor}" -eq 1 ]; then
+    before_previous_major=$((binary_major - 1))
+    before_previous_minor=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \
+    | grep --only-matching --perl-regexp "(?<=v)${before_previous_major}.\d.[\d]+?(?=[\^])" \
+    | sort --numeric-sort --key 1.3 | tail -1 | cut -d '.' -f 2)
   fi
 
   # This gets a list of all remote tags for the release branch in regex
   # Sort key is used to sort numerically by patch version
   # Latest version is then stored for use below
-  UPGRADE_VER=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \
-    | grep --only-matching --perl-regexp "(?<=v)${binary_major}.${previous_minor}.[\d]+?(?=[\^])" \
+  UPGRADE_VER_LAST=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \
+    | grep --only-matching --perl-regexp "(?<=v)${previous_major}.${previous_minor}.[\d]+?(?=[\^])" \
+    | sort --numeric-sort --key 1.5 | tail -1 | sed 's/^/v/')
+  log_callout "Found latest release: ${UPGRADE_VER_LAST}."
+
+  if [ -n "${MANUAL_VER_LAST:-}" ]; then
+    # in case, we need to test against different version
+    UPGRADE_VER_LAST=$MANUAL_VER_LAST
+  fi
+  if [[ -z ${UPGRADE_VER_LAST} ]]; then
+    UPGRADE_VER_LAST="v3.5.0"
+    log_warning "fallback to" ${UPGRADE_VER_LAST}
+  fi
+
+  UPGRADE_VER_BEFORE_LAST=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \
+    | grep --only-matching --perl-regexp "(?<=v)${before_previous_major}.${before_previous_minor}.[\d]+?(?=[\^])" \
     | sort --numeric-sort --key 1.5 | tail -1 | sed 's/^/v/')
-  log_callout "Found latest release: ${UPGRADE_VER}."
+  log_callout "Found before latest release: ${UPGRADE_VER_BEFORE_LAST}."
 
-  if [ -n "${MANUAL_VER:-}" ]; then
+  if [ -n "${MANUAL_VER_BEFORE_LAST:-}" ]; then
     # in case, we need to test against different version
-    UPGRADE_VER=$MANUAL_VER
+    UPGRADE_VER_BEFORE_LAST=MANUAL_VER_BEFORE_LAST
   fi
-  if [[ -z ${UPGRADE_VER} ]]; then
-    UPGRADE_VER="v3.5.0"
-    log_warning "fallback to" ${UPGRADE_VER}
+  if [[ -z ${UPGRADE_VER_BEFORE_LAST} ]]; then
+    UPGRADE_VER_BEFORE_LAST="v3.4.0"
+    log_warning "fallback to" ${UPGRADE_VER_BEFORE_LAST}
   fi
 
-  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
-  log_callout "Downloading $file"
+  download_etcd_ver_to_tmp ${UPGRADE_VER_LAST}
+  mv /tmp/etcd ./bin/etcd-last-release
+
+  download_etcd_ver_to_tmp ${UPGRADE_VER_BEFORE_LAST}
+  mv /tmp/etcd ./bin/etcd-before-last-release
+}
+
+function download_etcd_ver_to_tmp {
+    local version="$1"
+    local file="etcd-$version-linux-$GOARCH.tar.gz"
+    log_callout "Downloading $file"
 
   set +e
-  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
+  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$version/$file" -o "/tmp/$file"
   local result=$?
   set -e
   case $result in
@@ -612,8 +650,6 @@ function release_pass {
   esac
 
   tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
-  mkdir -p ./bin
-  mv /tmp/etcd ./bin/etcd-last-release
 }
 
 function mod_tidy_for_module {

server/storage/schema/bucket.go

@@ -68,17 +68,19 @@ func (b bucket) String() string          { return string(b.Name()) }
 func (b bucket) IsSafeRangeBucket() bool { return b.safeRangeBucket }
 
 var (
-	// Pre v3.5
-	ScheduledCompactKeyName    = []byte("scheduledCompactRev")
-	FinishedCompactKeyName     = []byte("finishedCompactRev")
-	MetaConsistentIndexKeyName = []byte("consistent_index")
-	AuthEnabledKeyName         = []byte("authEnabled")
-	AuthRevisionKeyName        = []byte("authRevision")
-	// Since v3.5
-	MetaTermKeyName              = []byte("term")
-	MetaConfStateName            = []byte("confState")
+	// Pre v3.4
+	ScheduledCompactKeyName      = []byte("scheduledCompactRev")
+	FinishedCompactKeyName       = []byte("finishedCompactRev")
+	MetaConsistentIndexKeyName   = []byte("consistent_index")
+	AuthEnabledKeyName           = []byte("authEnabled")
+	AuthRevisionKeyName          = []byte("authRevision")
 	ClusterClusterVersionKeyName = []byte("clusterVersion")
-	ClusterDowngradeKeyName      = []byte("downgrade")
+
+	// No new keys were added in v3.4
+	// Since v3.5
+	MetaTermKeyName         = []byte("term")
+	MetaConfStateName       = []byte("confState")
+	ClusterDowngradeKeyName = []byte("downgrade")
 	// Since v3.6
 	MetaStorageVersionName = []byte("storageVersion")
 	// Before adding new meta key please update server/etcdserver/version

server/storage/schema/confstate_test.go

@@ -59,6 +59,22 @@ func TestMustUnsafeSaveConfStateToBackend(t *testing.T) {
 		assert.Nil(t, UnsafeConfStateFromBackend(lg, tx))
 	})
 
+	emptyConfState := raftpb.ConfState{}
+	t.Run("save empty", func(t *testing.T) {
+		tx := be.BatchTx()
+		tx.Lock()
+		MustUnsafeSaveConfStateToBackend(lg, tx, &emptyConfState)
+		tx.Unlock()
+		tx.Commit()
+	})
+
+	t.Run("read empty", func(t *testing.T) {
+		tx := be.ReadTx()
+		tx.RLock()
+		defer tx.RUnlock()
+		assert.Equal(t, emptyConfState, *UnsafeConfStateFromBackend(lg, tx))
+	})
+
 	confState := raftpb.ConfState{Learners: []uint64{1, 2}, Voters: []uint64{3}, AutoLeave: false}
 
 	t.Run("save", func(t *testing.T) {

server/storage/schema/membership.go

@@ -183,7 +183,7 @@ func mustParseMemberIDFromBytes(lg *zap.Logger, key []byte) types.ID {
 }
 
 // ClusterVersionFromBackend reads cluster version from backend.
-// The field is populated since etcd v3.5.
+// The field is populated since etcd v3.4.
 func (s *membershipBackend) ClusterVersionFromBackend() *semver.Version {
 	ckey := ClusterClusterVersionKeyName
 	tx := s.be.ReadTx()

server/storage/schema/membership_test.go

@@ -0,0 +1,46 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"go.uber.org/zap/zaptest"
+
+	"go.etcd.io/etcd/api/v3/version"
+	serverversion "go.etcd.io/etcd/server/v3/etcdserver/version"
+	betesting "go.etcd.io/etcd/server/v3/storage/backend/testing"
+)
+
+func TestDowngradeInfoFromBackend(t *testing.T) {
+	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+
+	mbe := NewMembershipBackend(lg, be)
+
+	mbe.MustCreateBackendBuckets()
+	mbe.be.ForceCommit()
+	assert.Nil(t, mbe.DowngradeInfoFromBackend())
+
+	dinfo := &serverversion.DowngradeInfo{Enabled: true, TargetVersion: version.V3_5.String()}
+	mbe.MustSaveDowngradeToBackend(dinfo)
+
+	info := mbe.DowngradeInfoFromBackend()
+
+	assert.Equal(t, dinfo, info)
+}

server/storage/schema/schema.go

@@ -15,14 +15,17 @@
 package schema
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/coreos/go-semver/semver"
+
 	"go.uber.org/zap"
 
 	"go.etcd.io/etcd/api/v3/version"
-
+	serverversion "go.etcd.io/etcd/server/v3/etcdserver/version"
 	"go.etcd.io/etcd/server/v3/storage/backend"
+	"go.etcd.io/raft/v3/raftpb"
 )
 
 // Validate checks provided backend to confirm that schema used is supported.
@@ -83,7 +86,7 @@ func UnsafeMigrate(lg *zap.Logger, tx backend.UnsafeReadWriter, w WALVersion, ta
 // DetectSchemaVersion returns version of storage schema. Returned value depends on etcd version that created the backend. For
 // * v3.6 and newer will return storage version.
 // * v3.5 will return it's version if it includes all storage fields added in v3.5 (might require a snapshot).
-// * v3.4 and older is not supported and will return error.
+// * v3.4 will return it's version if it doesn't include all storage fields added in v3.5.
 func DetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, err error) {
 	tx.RLock()
 	defer tx.RUnlock()
@@ -96,12 +99,18 @@ func UnsafeDetectSchemaVersion(lg *zap.Logger, tx backend.UnsafeReader) (v semve
 	if vp != nil {
 		return *vp, nil
 	}
+
 	confstate := UnsafeConfStateFromBackend(lg, tx)
-	if confstate == nil {
-		return v, fmt.Errorf("missing confstate information")
-	}
 	_, term := UnsafeReadConsistentIndex(tx)
-	if term == 0 {
+	if confstate == nil && term == 0 {
+		// if both confstate and term are missing, assume it's v3.4
+		return version.V3_4, nil
+	} else if confstate == nil {
+		return v, fmt.Errorf("missing confstate information")
+	} else if len(confstate.Voters) == 0 && term == 0 {
+		// if confstate is empty and term is missing, assume it's v3.5 that was migrated from v3.4 and never started
+		return version.V3_5, nil
+	} else if term == 0 {
 		return v, fmt.Errorf("missing term information")
 	}
 	return version.V3_5, nil
@@ -129,10 +138,21 @@ var (
 	// schema was introduced in v3.6 as so its changes were not tracked before.
 	schemaChanges = map[semver.Version][]schemaChange{
 		version.V3_6: {
-			addNewField(Meta, MetaStorageVersionName, emptyStorageVersion),
+			// emptyValue is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator.
+			addNewField(Meta, MetaStorageVersionName, emptyValue),
+		},
+		version.V3_5: {
+			// UnsafeReadConsistentIndex will fail on []byte(""), use 0 as default
+			addNewField(Meta, MetaTermKeyName, emptyTerm),
+			// UnsafeConfStateFromBackend will fail on []byte(""), use empty struct as default
+			addNewField(Meta, MetaConfStateName, emptyConfState),
+			// DowngradeInfoFromBackend will fail on []byte(""), false is better default
+			addNewField(Cluster, ClusterDowngradeKeyName, falseDowngradeInfo),
 		},
 	}
-	// emptyStorageVersion is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator.
-	// Adding a addNewField for StorageVersion we can reuse logic to remove it when downgrading to v3.5
-	emptyStorageVersion = []byte("")
+
+	emptyValue            = []byte("")
+	emptyTerm             = make([]byte, 8)
+	emptyConfState, _     = json.Marshal(raftpb.ConfState{})
+	falseDowngradeInfo, _ = json.Marshal(serverversion.DowngradeInfo{Enabled: false, TargetVersion: ""})
 )