diff --git a/tests/robustness/README.md b/tests/robustness/README.md index f8915a85d0c..d5d71c32985 100644 --- a/tests/robustness/README.md +++ b/tests/robustness/README.md @@ -19,7 +19,7 @@ The purpose of these tests is to rigorously validate that etcd maintains its [KV | Duplicated watch event due to bug in TXN caching [#17247] | Jan 2024 | main branch | Robustness | Yes, prevented regression in v3.6 | | | Watch events lost during stream starvation [#17529] | Mar 2024 | v3.4 or earlier | User | Yes, after covering of slow watch | `make test-robustness-issue17529` | | Revision decreasing caused by crash during compaction [#17780] | Apr 2024 | v3.4 or earlier | Robustness | Yes, after covering compaction | | -| Watch dropping an event when compacting on delete [#18089] | May 2024 | v3.4 or earlier | Robustness | Yes, after covering of compaction | | +| Watch dropping an event when compacting on delete [#18089] | May 2024 | v3.4 or earlier | Robustness | Yes, after covering of compaction | `make test-robustness-issue18089` | | Inconsistency when reading compacted revision in TXN [#18667] | Oct 2024 | v3.4 or earlier | User | | | [#13766]: https://github.com/etcd-io/etcd/issues/13766 diff --git a/tests/robustness/main_test.go b/tests/robustness/main_test.go index 32e03283820..11da8bdcdee 100644 --- a/tests/robustness/main_test.go +++ b/tests/robustness/main_test.go @@ -41,7 +41,7 @@ var testRunner = framework.E2eTestRunner var ( WaitBeforeFailpoint = time.Second - WaitJitter = traffic.CompactionPeriod + WaitJitter = traffic.DefaultCompactionPeriod WaitAfterFailpoint = time.Second ) diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk index 56108976c7e..94b3aa47b28 100644 --- a/tests/robustness/makefile.mk +++ b/tests/robustness/makefile.mk @@ -49,6 +49,11 @@ test-robustness-issue17780: /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bi GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 200 --failfast --bin-dir=/tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" +.PHONY: test-robustness-issue18089 +test-robustness-issue18089: /tmp/etcd-v3.5.12-beforeSendWatchResponse/bin + GO_TEST_FLAGS='-v -run=TestRobustnessRegression/Issue18089 -count 100 -failfast --bin-dir=/tmp/etcd-v3.5.12-beforeSendWatchResponse/bin' make test-robustness && \ + echo "Failed to reproduce" || echo "Successful reproduction" + # Failpoints GOPATH = $(shell go env GOPATH) diff --git a/tests/robustness/scenarios/scenarios.go b/tests/robustness/scenarios/scenarios.go index afad5879fee..8efcb48c5a6 100644 --- a/tests/robustness/scenarios/scenarios.go +++ b/tests/robustness/scenarios/scenarios.go @@ -223,6 +223,16 @@ func Regression(t *testing.T) []TestScenario { e2e.WithGoFailEnabled(true), ), }) + scenarios = append(scenarios, TestScenario{ + Name: "Issue18089", + Profile: traffic.LowTraffic.WithCompactionPeriod(100 * time.Millisecond), // Use frequent compaction for high reproduce rate + Failpoint: failpoint.SleepBeforeSendWatchResponse, + Traffic: traffic.EtcdDelete, + Cluster: *e2e.NewConfig( + e2e.WithClusterSize(1), + e2e.WithGoFailEnabled(true), + ), + }) if v.Compare(version.V3_5) >= 0 { opts := []e2e.EPClusterOption{ e2e.WithSnapshotCount(100), diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go index d8823ecd3e4..c426064f68f 100644 --- a/tests/robustness/traffic/etcd.go +++ b/tests/robustness/traffic/etcd.go @@ -64,6 +64,16 @@ var ( {Choice: Put, Weight: 40}, }, } + EtcdDelete Traffic = etcdTraffic{ + keyCount: 10, + largePutSize: 32769, + leaseTTL: DefaultLeaseTTL, + // Please keep the sum of weights equal 100. + requests: []random.ChoiceWeight[etcdRequestType]{ + {Choice: Put, Weight: 50}, + {Choice: Delete, Weight: 50}, + }, + } ) type etcdTraffic struct { diff --git a/tests/robustness/traffic/traffic.go b/tests/robustness/traffic/traffic.go index c2de307218b..09a59d30a7a 100644 --- a/tests/robustness/traffic/traffic.go +++ b/tests/robustness/traffic/traffic.go @@ -33,11 +33,11 @@ import ( ) var ( - DefaultLeaseTTL int64 = 7200 - RequestTimeout = 200 * time.Millisecond - WatchTimeout = time.Second - MultiOpTxnOpCount = 4 - CompactionPeriod = 200 * time.Millisecond + DefaultLeaseTTL int64 = 7200 + RequestTimeout = 200 * time.Millisecond + WatchTimeout = time.Second + MultiOpTxnOpCount = 4 + DefaultCompactionPeriod = 200 * time.Millisecond LowTraffic = Profile{ MinimalQPS: 100, @@ -96,7 +96,11 @@ func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2 defer wg.Done() defer c.Close() - RunCompactLoop(ctx, c, CompactionPeriod, finish) + compactionPeriod := DefaultCompactionPeriod + if profile.CompactPeriod != time.Duration(0) { + compactionPeriod = profile.CompactPeriod + } + RunCompactLoop(ctx, c, compactionPeriod, finish) mux.Lock() reports = append(reports, c.Report()) mux.Unlock() @@ -176,6 +180,7 @@ type Profile struct { MaxNonUniqueRequestConcurrency int ClientCount int ForbidCompaction bool + CompactPeriod time.Duration } func (p Profile) WithoutCompaction() Profile { @@ -183,6 +188,11 @@ func (p Profile) WithoutCompaction() Profile { return p } +func (p Profile) WithCompactionPeriod(cp time.Duration) Profile { + p.CompactPeriod = cp + return p +} + type Traffic interface { Run(ctx context.Context, c *client.RecordingClient, qpsLimiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIDStorage, nonUniqueWriteLimiter ConcurrencyLimiter, finish <-chan struct{}) ExpectUniqueRevision() bool