panic: version does not exist

[SynergyNodes]

Hello, this issue happened twice with Archive node, after a restart of story binary. Before getting this error, I have restarted both story and geth a few times, and there were no issues. I have added the logs below.

System Details:
Ubuntu 22.04
AMD Ryzen 9 7950X3D 16-Core Processor
NVM.e

Geth
Version: 0.9.3-stable

Story
Version v0.10.1-stable

Please let me know if you need any other details.

Regards,
Aries

24-10-10 03:37:18.184 INFO Starting story consensus client
24-10-10 03:37:18.185 INFO Version info                             version=v0.10.1-stable git_commit=57567e5 git_timestamp=2024-09-29T21:03:15Z
24-10-10 03:37:18.662 INFO 👾 ABCI call: Info
24-10-10 03:37:19.100 INFO 👾 ABCI response: FinalizeBlock          val_updates=0 height=1283411
24-10-10 03:37:19.100 INFO 👾 ABCI call: Commit
panic: version does not exist
goroutine 1 [running]:
cosmossdk.io/store/iavl.(*Store).Commit(0xc00609e990?)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/iavl/store.go:128 +0xd7
cosmossdk.io/store/rootmulti.commitStores(0x139553, 0xc005633170, 0xc00388d050)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/rootmulti/store.go:1196 +0x158
cosmossdk.io/store/rootmulti.(*Store).Commit(0xc0037ad200)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/rootmulti/store.go:482 +0x1a5
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Commit(0xc0001e6488)
	/home/runner/go/pkg/mod/github.com/cosmos/[email protected]/baseapp/abci.go:930 +0x2e2
github.com/cosmos/cosmos-sdk/server.cometABCIWrapper.Commit(...)
	/home/runner/go/pkg/mod/github.com/cosmos/[email protected]/server/cmt_abci.go:56
github.com/piplabs/story/client/app.abciWrapper.Commit({{0x375bf10?, 0xc005effa90?}, 0xc005effaa0?, 0xc005effab0?}, {0x3740988, 0x51d28e0}, 0x51d28e0)
	/home/runner/work/story/story/client/app/abci.go:163 +0x8a
github.com/cometbft/cometbft/abci/client.(*localClient).Commit(0x20?, {0x3740988?, 0x51d28e0?}, 0xc01030bdf8?)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/abci/client/local_client.go:113 +0xcd
github.com/cometbft/cometbft/proxy.(*appConnConsensus).Commit(0xc00391d488, {0x3740988, 0x51d28e0})
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/proxy/app_conn.go:109 +0x168
github.com/cometbft/cometbft/state.(*BlockExecutor).Commit(_, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, 0x1, 0x139553, {{0xc004a284e0, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:404 +0x213
github.com/cometbft/cometbft/state.(*BlockExecutor).applyBlock(_, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, 0x1, 0x139553, {{0xc004a284e0, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:290 +0xf10
github.com/cometbft/cometbft/state.(*BlockExecutor).ApplyBlock(_, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, 0x1, 0x139552, {{0xc004a28100, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:219 +0x1d9
github.com/cometbft/cometbft/consensus.(*Handshaker).replayBlock(_, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, 0x1, 0x139552, {{0xc004a28100, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:534 +0x24e
github.com/cometbft/cometbft/consensus.(*Handshaker).ReplayBlocksWithContext(_, {_, _}, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, 0x1, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:433 +0x77a
github.com/cometbft/cometbft/consensus.(*Handshaker).HandshakeWithContext(0xc01030d988, {0x3740988, 0x51d28e0}, {0x375c498, 0xc001008000})
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:274 +0x425
github.com/cometbft/cometbft/node.doHandshake({_, _}, {_, _}, {{{0xb, 0x0}, {0xc004a262c8, 0x6}}, {0xc004a262f0, 0x7}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/setup.go:182 +0x1a9
github.com/cometbft/cometbft/node.NewNodeWithContext({0x3740988, 0x51d28e0}, 0xc000cf90d0, {0x37293a0, 0xc00087a780}, 0xc005effa50, {0x3713500, 0xc0069ec1c8}, 0xc0047487b8, 0x30ca498, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/node.go:359 +0x5b7
github.com/cometbft/cometbft/node.NewNode(0x3fe6a9fe36b284ae?, {0x37293a0?, 0xc00087a780?}, 0x59c29285cbefa24b?, {0x3713500?, 0xc0069ec1c8?}, 0x90d97de1559fcdaf?, 0xc258660a5f092569?, 0x68bc1335e1dfcd54?, {0x3740a68, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/node.go:279 +0xa9
github.com/piplabs/story/client/app.newCometNode({0x37408a8, 0xc000d4d9e0}, 0xc000cf90d0, 0xc000a68c08, {0x37293a0, 0xc00087a780})
	/home/runner/work/story/story/client/app/start.go:207 +0x49f
github.com/piplabs/story/client/app.Start({_, _}, {{{0xc0011cba58, 0x17}, {0xc000015a00, 0x5}, {0x0, 0x0}, {0xc001706e40, 0x2b}, ...}, ...})
	/home/runner/work/story/story/client/app/start.go:133 +0x625
github.com/piplabs/story/client/app.Run({_, _}, {{{0xc0011cba58, 0x17}, {0xc000015a00, 0x5}, {0x0, 0x0}, {0xc001706e40, 0x2b}, ...}, ...})
	/home/runner/work/story/story/client/app/start.go:56 +0x79
github.com/piplabs/story/client/cmd.newRunCmd.func1(0xc000a6cc08, {0x2d00332?, 0x4?, 0x2d0034e?})
	/home/runner/work/story/story/client/cmd/cmd.go:51 +0x225
github.com/spf13/cobra.(*Command).execute(0xc000a6cc08, {0x51d28e0, 0x0, 0x0})
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:983 +0xaca
github.com/spf13/cobra.(*Command).ExecuteC(0xc00077f808)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1115 +0x3ff
github.com/spf13/cobra.(*Command).Execute(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1039
github.com/spf13/cobra.(*Command).ExecuteContext(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1032
github.com/piplabs/story/lib/cmd.Main(0xc00077f808)
	/home/runner/work/story/story/lib/cmd/cmd.go:34 +0xcf
main.main()
	/home/runner/work/story/story/client/main.go:10 +0x18```

[1scrooge]

I have the same problem 5 times on different nodes.
story version is v0.13.0

25-01-16 12:08:03.980 INFO Starting story consensus client
25-01-16 12:08:03.981 INFO Version info                             version=v0.13.0-stable git_commit=daaa395 git_timestamp=2024-11-19T03:59:16Z
25-01-16 12:08:05.788 INFO 👾 ABCI call: Info
25-01-16 12:08:05.822 INFO 👾 ABCI call: FinalizeBlock              height=2021300 proposer=048feda
25-01-16 12:08:06.180 INFO 👾 ABCI response: FinalizeBlock          val_updates=0 height=2021300
25-01-16 12:08:06.186 INFO 👾 ABCI call: Commit
panic: version does not exist

goroutine 1 [running]:
cosmossdk.io/store/iavl.(*Store).Commit(0xc00567c6f0?)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/iavl/store.go:128 +0xd7
cosmossdk.io/store/rootmulti.commitStores(0x1ed7b4, 0xc004e68ba0, 0xc004a86690)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/rootmulti/store.go:1196 +0x158
cosmossdk.io/store/rootmulti.(*Store).Commit(0xc004a09000)
	/home/runner/go/pkg/mod/cosmossdk.io/[email protected]/rootmulti/store.go:482 +0x1a5
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Commit(0xc000b8e488)
	/home/runner/go/pkg/mod/github.com/piplabs/[email protected]/baseapp/abci.go:930 +0x2e2
github.com/cosmos/cosmos-sdk/server.cometABCIWrapper.Commit(...)
	/home/runner/go/pkg/mod/github.com/piplabs/[email protected]/server/cmt_abci.go:56
github.com/piplabs/story/client/app.abciWrapper.Commit({{0x379d1b8?, 0xc004bdc7f0?}, 0xc004bdc800?, 0xc004bdc810?}, {0x37819b8, 0x52369c0}, 0x52369c0)
	/home/runner/work/story/story/client/app/abci.go:167 +0x8a
github.com/cometbft/cometbft/abci/client.(*localClient).Commit(0x20?, {0x37819b8?, 0x52369c0?}, 0xc00828bc38?)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/abci/client/local_client.go:113 +0xcd
github.com/cometbft/cometbft/proxy.(*appConnConsensus).Commit(0xc00587d0f8, {0x37819b8, 0x52369c0})
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/proxy/app_conn.go:109 +0x168
github.com/cometbft/cometbft/state.(*BlockExecutor).Commit(_, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, 0x1, 0x1ed7b4, {{0xc003eb80c0, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:404 +0x213
github.com/cometbft/cometbft/state.(*BlockExecutor).applyBlock(_, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, 0x1, 0x1ed7b4, {{0xc003eb80c0, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:290 +0xf10
github.com/cometbft/cometbft/state.(*BlockExecutor).ApplyBlock(_, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, 0x1, 0x1ed7b3, {{0xc002c8ed40, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/state/execution.go:219 +0x1d9
github.com/cometbft/cometbft/consensus.(*Handshaker).replayBlock(_, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, 0x1, 0x1ed7b3, {{0xc002c8ed40, ...}, ...}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:534 +0x24e
github.com/cometbft/cometbft/consensus.(*Handshaker).ReplayBlocksWithContext(_, {_, _}, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, 0x1, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:433 +0x77a
github.com/cometbft/cometbft/consensus.(*Handshaker).HandshakeWithContext(0xc00828d7c8, {0x37819b8, 0x52369c0}, {0x379d518, 0xc0005d3dc0})
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/consensus/replay.go:274 +0x425
github.com/cometbft/cometbft/node.doHandshake({_, _}, {_, _}, {{{0xb, 0x0}, {0xc003c10d28, 0x6}}, {0xc003c10d50, 0x9}, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/setup.go:182 +0x1a9
github.com/cometbft/cometbft/node.NewNodeWithContext({0x37819b8, 0x52369c0}, 0xc000ca47d0, {0x376a360, 0xc000e22000}, 0xc004bdc7b0, {0x37544c0, 0xc004c1f650}, 0xc0038525f8, 0x30fc078, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/node.go:359 +0x5b7
github.com/cometbft/cometbft/node.NewNode(0x5431c4b6c5420f74?, {0x376a360?, 0xc000e22000?}, 0xf45f6dfde0e9fd67?, {0x37544c0?, 0xc004c1f650?}, 0x4148848f904bc392?, 0x1a0987160af0706f?, 0xa1abb25a14230b01?, {0x3781b78, ...}, ...)
	/home/runner/go/pkg/mod/github.com/cometbft/[email protected]/node/node.go:279 +0xa9
github.com/piplabs/story/client/app.newCometNode({0x37818d8, 0xc000c35ef0}, 0xc000ca47d0, 0xc000564b08, {0x376a360, 0xc000e22000})
	/home/runner/work/story/story/client/app/start.go:251 +0x49f
github.com/piplabs/story/client/app.Start({_, _}, {{{0xc000e8f728, 0x12}, {0xc000bcabe0, 0x7}, {0x0, 0x0}, {0xc0001354d0, 0x28}, ...}, ...})
	/home/runner/work/story/story/client/app/start.go:133 +0x625
github.com/piplabs/story/client/app.Run({_, _}, {{{0xc000e8f728, 0x12}, {0xc000bcabe0, 0x7}, {0x0, 0x0}, {0xc0001354d0, 0x28}, ...}, ...})
	/home/runner/work/story/story/client/app/start.go:56 +0x79
github.com/piplabs/story/client/cmd.newRunCmd.func1(0xc000c24608, {0x2d36006?, 0x4?, 0x2d36046?})
	/home/runner/work/story/story/client/cmd/cmd.go:56 +0x225
github.com/spf13/cobra.(*Command).execute(0xc000c24608, {0x52369c0, 0x0, 0x0})
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:985 +0xaca
github.com/spf13/cobra.(*Command).ExecuteC(0xc000c3e008)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1117 +0x3ff
github.com/spf13/cobra.(*Command).Execute(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1041
github.com/spf13/cobra.(*Command).ExecuteContext(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/command.go:1034
github.com/piplabs/story/lib/cmd.Main(0xc000c3e008)
	/home/runner/work/story/story/lib/cmd/cmd.go:34 +0xcf
main.main()
	/home/runner/work/story/story/client/main.go:10 +0x18