diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go index 2253fc79a36..3ce2ff39a53 100644 --- a/tests/robustness/failpoint/gofail.go +++ b/tests/robustness/failpoint/gofail.go @@ -31,36 +31,37 @@ import ( ) var ( - DefragBeforeCopyPanic Failpoint = goPanicFailpoint{"defragBeforeCopy", triggerDefrag{}, AnyMember} - DefragBeforeRenamePanic Failpoint = goPanicFailpoint{"defragBeforeRename", triggerDefrag{}, AnyMember} - BeforeCommitPanic Failpoint = goPanicFailpoint{"beforeCommit", nil, AnyMember} - AfterCommitPanic Failpoint = goPanicFailpoint{"afterCommit", nil, AnyMember} - RaftBeforeSavePanic Failpoint = goPanicFailpoint{"raftBeforeSave", nil, AnyMember} - RaftAfterSavePanic Failpoint = goPanicFailpoint{"raftAfterSave", nil, AnyMember} - BackendBeforePreCommitHookPanic Failpoint = goPanicFailpoint{"commitBeforePreCommitHook", nil, AnyMember} - BackendAfterPreCommitHookPanic Failpoint = goPanicFailpoint{"commitAfterPreCommitHook", nil, AnyMember} - BackendBeforeStartDBTxnPanic Failpoint = goPanicFailpoint{"beforeStartDBTxn", nil, AnyMember} - BackendAfterStartDBTxnPanic Failpoint = goPanicFailpoint{"afterStartDBTxn", nil, AnyMember} - BackendBeforeWritebackBufPanic Failpoint = goPanicFailpoint{"beforeWritebackBuf", nil, AnyMember} - BackendAfterWritebackBufPanic Failpoint = goPanicFailpoint{"afterWritebackBuf", nil, AnyMember} - CompactBeforeCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactBeforeCommitScheduledCompact", triggerCompact{}, AnyMember} - CompactAfterCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactAfterCommitScheduledCompact", triggerCompact{}, AnyMember} - CompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactBeforeSetFinishedCompact", triggerCompact{}, AnyMember} - CompactAfterSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactAfterSetFinishedCompact", triggerCompact{}, AnyMember} - CompactBeforeCommitBatchPanic Failpoint = goPanicFailpoint{"compactBeforeCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} - CompactAfterCommitBatchPanic Failpoint = goPanicFailpoint{"compactAfterCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} - RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader} - RaftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, Follower} - RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} - RaftAfterApplySnapPanic Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} - RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackhole{waitTillSnapshot: true}, Follower} - RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} - RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} - ApplyBeforeOpenSnapshot Failpoint = goPanicFailpoint{"applyBeforeOpenSnapshot", triggerBlackhole{waitTillSnapshot: true}, Follower} - BeforeApplyOneConfChangeSleep Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second} - RaftBeforeSaveSleep Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second} - RaftAfterSaveSleep Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second} - SleepBeforeSendWatchResponse Failpoint = gofailSleepAndDeactivate{"beforeSendWatchResponse", time.Second} + DefragBeforeCopyPanic Failpoint = goPanicFailpoint{"defragBeforeCopy", triggerDefrag{}, AnyMember} + DefragBeforeRenamePanic Failpoint = goPanicFailpoint{"defragBeforeRename", triggerDefrag{}, AnyMember} + BeforeCommitPanic Failpoint = goPanicFailpoint{"beforeCommit", nil, AnyMember} + AfterCommitPanic Failpoint = goPanicFailpoint{"afterCommit", nil, AnyMember} + RaftBeforeSavePanic Failpoint = goPanicFailpoint{"raftBeforeSave", nil, AnyMember} + RaftAfterSavePanic Failpoint = goPanicFailpoint{"raftAfterSave", nil, AnyMember} + BackendBeforePreCommitHookPanic Failpoint = goPanicFailpoint{"commitBeforePreCommitHook", nil, AnyMember} + BackendAfterPreCommitHookPanic Failpoint = goPanicFailpoint{"commitAfterPreCommitHook", nil, AnyMember} + BackendBeforeStartDBTxnPanic Failpoint = goPanicFailpoint{"beforeStartDBTxn", nil, AnyMember} + BackendAfterStartDBTxnPanic Failpoint = goPanicFailpoint{"afterStartDBTxn", nil, AnyMember} + BackendBeforeWritebackBufPanic Failpoint = goPanicFailpoint{"beforeWritebackBuf", nil, AnyMember} + BackendAfterWritebackBufPanic Failpoint = goPanicFailpoint{"afterWritebackBuf", nil, AnyMember} + CompactBeforeCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactBeforeCommitScheduledCompact", triggerCompact{}, AnyMember} + CompactAfterCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactAfterCommitScheduledCompact", triggerCompact{}, AnyMember} + CompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactBeforeSetFinishedCompact", triggerCompact{}, AnyMember} + BatchCompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactBeforeSetFinishedCompact", triggerCompact{multiBatchCompaction: true}, AnyMember} + CompactAfterSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactAfterSetFinishedCompact", triggerCompact{}, AnyMember} + CompactBeforeCommitBatchPanic Failpoint = goPanicFailpoint{"compactBeforeCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} + CompactAfterCommitBatchPanic Failpoint = goPanicFailpoint{"compactAfterCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} + RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader} + RaftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, Follower} + RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} + RaftAfterApplySnapPanic Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} + RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackhole{waitTillSnapshot: true}, Follower} + RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} + RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} + ApplyBeforeOpenSnapshot Failpoint = goPanicFailpoint{"applyBeforeOpenSnapshot", triggerBlackhole{waitTillSnapshot: true}, Follower} + BeforeApplyOneConfChangeSleep Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second} + RaftBeforeSaveSleep Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second} + RaftAfterSaveSleep Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second} + SleepBeforeSendWatchResponse Failpoint = gofailSleepAndDeactivate{"beforeSendWatchResponse", time.Second} ) type goPanicFailpoint struct { diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk index 09d10108fb6..56108976c7e 100644 --- a/tests/robustness/makefile.mk +++ b/tests/robustness/makefile.mk @@ -44,6 +44,11 @@ test-robustness-issue17529: /tmp/etcd-v3.5.12-beforeSendWatchResponse/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17529 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.12-beforeSendWatchResponse/bin' $(MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" +.PHONY: test-robustness-issue17780 +test-robustness-issue17780: /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 200 --failfast --bin-dir=/tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin' make test-robustness && \ + echo "Failed to reproduce" || echo "Successful reproduction" + # Failpoints GOPATH = $(shell go env GOPATH) @@ -119,6 +124,20 @@ $(GOPATH)/bin/gofail: tools/mod/go.mod tools/mod/go.sum (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; +/tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin: $(GOPATH)/bin/gofail + rm -rf /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ + mkdir -p /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ + git clone --depth 1 --branch v3.5.13 https://github.com/etcd-io/etcd.git /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ + cp -r ./tests/robustness/patches/compactBeforeSetFinishedCompact /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ + cd /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/; \ + patch -l server/mvcc/kvstore_compaction.go ./compactBeforeSetFinishedCompact/kvstore_compaction.patch; \ + go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ + (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ + FAILPOINTS=true ./build; + /tmp/etcd-release-3.5-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-release-3.5-failpoints/ mkdir -p /tmp/etcd-release-3.5-failpoints/ diff --git a/tests/robustness/patches/compactBeforeSetFinishedCompact/kvstore_compaction.patch b/tests/robustness/patches/compactBeforeSetFinishedCompact/kvstore_compaction.patch new file mode 100644 index 00000000000..b04978b7c3e --- /dev/null +++ b/tests/robustness/patches/compactBeforeSetFinishedCompact/kvstore_compaction.patch @@ -0,0 +1,26 @@ +From 6b034466aa0ac2b46fe01fb5bd2233946f46d453 Mon Sep 17 00:00:00 2001 +From: Wei Fu +Date: Wed, 24 Apr 2024 12:14:27 +0800 +Subject: [PATCH] server/mvcc: introduce compactBeforeSetFinishedCompact + failpoint + +Signed-off-by: Wei Fu +--- + server/mvcc/kvstore_compaction.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/server/mvcc/kvstore_compaction.go b/server/mvcc/kvstore_compaction.go +index c7d343d5c..89defbd9e 100644 +--- a/server/mvcc/kvstore_compaction.go ++++ b/server/mvcc/kvstore_compaction.go +@@ -59,6 +59,7 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyVal + } + + if len(keys) < s.cfg.CompactionBatchLimit { ++ // gofail: var compactBeforeSetFinishedCompact struct{} + rbytes := make([]byte, 8+1+8) + revToBytes(revision{main: compactMainRev}, rbytes) + tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) +-- +2.34.1 + diff --git a/tests/robustness/scenarios/scenarios.go b/tests/robustness/scenarios/scenarios.go index 6034ee9b312..afad5879fee 100644 --- a/tests/robustness/scenarios/scenarios.go +++ b/tests/robustness/scenarios/scenarios.go @@ -210,6 +210,19 @@ func Regression(t *testing.T) []TestScenario { options.WithSnapshotCount(100), ), }) + + scenarios = append(scenarios, TestScenario{ + Name: "Issue17780", + Profile: traffic.LowTraffic.WithoutCompaction(), + Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, + Traffic: traffic.Kubernetes, + Cluster: *e2e.NewConfig( + e2e.WithClusterSize(1), + e2e.WithCompactionBatchLimit(300), + e2e.WithSnapshotCount(1000), + e2e.WithGoFailEnabled(true), + ), + }) if v.Compare(version.V3_5) >= 0 { opts := []e2e.EPClusterOption{ e2e.WithSnapshotCount(100),