From 8db476f189f9e22d111e46aaf8f07afb66ea54fc Mon Sep 17 00:00:00 2001
From: Unique Divine <51418232+Unique-Divine@users.noreply.github.com>
Date: Fri, 10 Jan 2025 14:27:00 -0600
Subject: [PATCH] test(testutil): try retrying for 'panic: pebbledb: closed'
 (#2162)

* test(testutil): try retrying for 'panic: pebbledb: closed'

* chore: changelog
---
 CHANGELOG.md                                  |  1 +
 app/wasmext/wasm_cli_test/cli_test.go         |  4 +-
 eth/rpc/rpcapi/eth_api_test.go                |  5 ++-
 x/common/testutil/cases.go                    | 38 +++++++++++++++++++
 x/common/testutil/testnetwork/network_test.go |  4 +-
 x/oracle/keeper/app_test.go                   |  4 +-
 x/sudo/cli/cli_test.go                        |  4 +-
 x/tokenfactory/cli/cli_test.go                |  4 +-
 8 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd3c85c40..cb709ee61 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -75,6 +75,7 @@ needed to include double quotes around the hexadecimal string.
 - [#2156](https://github.com/NibiruChain/nibiru/pull/2156) - test(evm-e2e): add E2E test using the Nibiru Oracle's ChainLink impl
 - [#2157](https://github.com/NibiruChain/nibiru/pull/2157) - fix(evm): Fix unit inconsistency related to AuthInfo.Fee and txData.Fee using effective fee
 - [#2160](https://github.com/NibiruChain/nibiru/pull/2160) - fix(evm-precompile): use bank.MsgServer Send in precompile IFunToken.bankMsgSend
+- [#2162](https://github.com/NibiruChain/nibiru/pull/2162) - test(testutil): try retrying for 'panic: pebbledb: closed'
 
 #### Nibiru EVM | Before Audit 2 - 2024-12-06
 
diff --git a/app/wasmext/wasm_cli_test/cli_test.go b/app/wasmext/wasm_cli_test/cli_test.go
index 2edc2fbb7..59ecc202f 100644
--- a/app/wasmext/wasm_cli_test/cli_test.go
+++ b/app/wasmext/wasm_cli_test/cli_test.go
@@ -140,5 +140,7 @@ func (s *TestSuite) requiredDeployedContractsLen(total int) {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
diff --git a/eth/rpc/rpcapi/eth_api_test.go b/eth/rpc/rpcapi/eth_api_test.go
index 3f274d1cf..cb42021ef 100644
--- a/eth/rpc/rpcapi/eth_api_test.go
+++ b/eth/rpc/rpcapi/eth_api_test.go
@@ -58,7 +58,10 @@ type NodeSuite struct {
 
 func TestSuite_RunAll(t *testing.T) {
 	suite.Run(t, new(Suite))
-	suite.Run(t, new(NodeSuite))
+
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(NodeSuite))
+	}, 2)
 }
 
 // SetupSuite runs before every test in the suite. Implements the
diff --git a/x/common/testutil/cases.go b/x/common/testutil/cases.go
index 547738d00..8ef1dc313 100644
--- a/x/common/testutil/cases.go
+++ b/x/common/testutil/cases.go
@@ -1,6 +1,7 @@
 package testutil
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -34,3 +35,40 @@ func BeforeIntegrationSuite(suiteT *testing.T) {
 	}
 	suiteT.Log("setting up integration test suite")
 }
+
+// RetrySuiteRunIfDbClosed runs a test suite with retries, recovering from a
+// specific panic message, "pebbledb: closed" that often surfaces in CI when tests
+// involve "Nibiru/x/common/testutil/testnetwork".
+// For full context, see https://github.com/NibiruChain/nibiru/issues/1918.
+func RetrySuiteRunIfDbClosed(t *testing.T, runTest func(), maxRetries int) {
+	panicMessage := "pebbledb: closed"
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		panicked := false
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					if errMsg, ok := r.(string); ok && strings.Contains(errMsg, panicMessage) {
+						t.Logf("Recovered from panic on attempt %d: %v", attempt, r)
+						panicked = true
+					} else {
+						panic(r) // Re-panic if it's not the specific error
+					}
+				}
+			}()
+
+			// Run the test suite
+			runTest()
+			// suite.Run(t, suiteInstance)
+		}()
+
+		if !panicked {
+			t.Logf("Test suite succeeded on attempt %d", attempt)
+			return
+		}
+
+		t.Logf("Retrying test suite: attempt %d", attempt+1)
+	}
+
+	t.Fatalf("Test suite failed after %d attempts due to '%s'", maxRetries, panicMessage)
+}
diff --git a/x/common/testutil/testnetwork/network_test.go b/x/common/testutil/testnetwork/network_test.go
index fa9fe2d8b..a36f2f658 100644
--- a/x/common/testutil/testnetwork/network_test.go
+++ b/x/common/testutil/testnetwork/network_test.go
@@ -22,7 +22,9 @@ import (
 )
 
 func TestIntegrationTestSuite_RunAll(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // Assert network cleanup
diff --git a/x/oracle/keeper/app_test.go b/x/oracle/keeper/app_test.go
index aac101285..197c73223 100644
--- a/x/oracle/keeper/app_test.go
+++ b/x/oracle/keeper/app_test.go
@@ -177,7 +177,9 @@ func (s *TestSuite) currentPrices() map[asset.Pair]sdk.Dec {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 func (s *TestSuite) TearDownSuite() {
diff --git a/x/sudo/cli/cli_test.go b/x/sudo/cli/cli_test.go
index 96baa5064..2dd167a18 100644
--- a/x/sudo/cli/cli_test.go
+++ b/x/sudo/cli/cli_test.go
@@ -96,7 +96,9 @@ type Account struct {
 }
 
 func TestSuite_IntegrationSuite_RunAll(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // ———————————————————————————————————————————————————————————————————
diff --git a/x/tokenfactory/cli/cli_test.go b/x/tokenfactory/cli/cli_test.go
index e80062e32..7680e99aa 100644
--- a/x/tokenfactory/cli/cli_test.go
+++ b/x/tokenfactory/cli/cli_test.go
@@ -31,7 +31,9 @@ type TestSuite struct {
 }
 
 func TestIntegrationTestSuite(t *testing.T) {
-	suite.Run(t, new(TestSuite))
+	testutil.RetrySuiteRunIfDbClosed(t, func() {
+		suite.Run(t, new(TestSuite))
+	}, 2)
 }
 
 // TestTokenFactory: Runs the test suite with a deterministic order.