From 8926deb09096b030f19d5f4ee31d76e59fc8487e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sat, 30 Sep 2023 16:16:50 +0200 Subject: [PATCH] Reproduce --wait-cluster-ready-timeout flag causing linearizability issue Signed-off-by: Marek Siarkowicz --- .github/workflows/robustness-template.yaml | 2 +- tests/robustness/failpoints.go | 17 +++++++++-------- tests/robustness/linearizability_test.go | 2 ++ tests/robustness/traffic/traffic.go | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/robustness-template.yaml b/.github/workflows/robustness-template.yaml index c668beaed780..0158242516b4 100644 --- a/.github/workflows/robustness-template.yaml +++ b/.github/workflows/robustness-template.yaml @@ -53,7 +53,7 @@ jobs: apt update && apt-get --yes install cmake libfuse3-dev libfuse3-3 fuse3 make install-lazyfs # Use --failfast to avoid overriding report generated by failed test - GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness" + GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness/Etcd/HighTraffic/ClusterOfSize1" case "${ETCD_BRANCH}" in release-3.5) EXPECT_DEBUG=true GO_TEST_FLAGS=${GO_TEST_FLAGS} RESULTS_DIR=/tmp/results make test-robustness-release-3.5 diff --git a/tests/robustness/failpoints.go b/tests/robustness/failpoints.go index d7fbba8aae8d..664d5220d702 100644 --- a/tests/robustness/failpoints.go +++ b/tests/robustness/failpoints.go @@ -67,14 +67,15 @@ var ( RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} allFailpoints = []Failpoint{ - KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, - DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, - BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, - BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, - CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic, - CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, - RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, - RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, + KillFailpoint, + //BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, + //DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, + //BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, + //BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, + //CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic, + //CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, + //RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, + //RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, } ) diff --git a/tests/robustness/linearizability_test.go b/tests/robustness/linearizability_test.go index e905ada83d42..27cecd9ca934 100644 --- a/tests/robustness/linearizability_test.go +++ b/tests/robustness/linearizability_test.go @@ -69,6 +69,8 @@ func TestRobustness(t *testing.T) { e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(100), e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), + e2e.WithWaitClusterReadyTimeout(time.Nanosecond), + e2e.WithEnvVars(map[string]string{"GOFAIL_FAILPOINTS": `beforeApplyOneConfChange=sleep("0.1s")`}), } scenarios := []testScenario{} for _, tp := range trafficProfiles { diff --git a/tests/robustness/traffic/traffic.go b/tests/robustness/traffic/traffic.go index 4d98d2c26529..75365b291da3 100644 --- a/tests/robustness/traffic/traffic.go +++ b/tests/robustness/traffic/traffic.go @@ -36,7 +36,7 @@ var ( LowTraffic = Profile{ Name: "LowTraffic", - MinimalQPS: 100, + MinimalQPS: 50, MaximalQPS: 200, ClientCount: 8, MaxNonUniqueRequestConcurrency: 3,