Merge branch 'main' into sim_utils

CHANGELOG.md

@@ -34,6 +34,8 @@
 - [#1088](https://github.com/crypto-org-chain/cronos/pull/1088) memiavl fix empty value in write-ahead-log replaying.
 - [#1102](https://github.com/crypto-org-chain/cronos/pull/1102) avoid duplicate cache events emitted from ibc and gravity hook.
 - [#1123](https://github.com/crypto-org-chain/cronos/pull/1123) Fix memiavl snapshot switching
+- [#1125](https://github.com/crypto-org-chain/cronos/pull/1125) Fix genesis migrate for feeibc, evm, feemarket and gravity.
+- [#1130](https://github.com/crypto-org-chain/cronos/pull/1130) Fix lock issues when state-sync with memiavl.
 
 ### Features
 
@@ -43,6 +45,7 @@
 - [#1100](https://github.com/crypto-org-chain/cronos/pull/1100) memiavl support read-only mode, and grab exclusive lock for write mode.
 - [#1103](https://github.com/crypto-org-chain/cronos/pull/1103) Add EventQueryTxFor cmd to subscribe and wait for transaction.
 - [#1108](https://github.com/crypto-org-chain/cronos/pull/1108) versiondb support restore from local snapshot.
+- [#1114](https://github.com/crypto-org-chain/cronos/pull/1114) memiavl support `CacheMultiStoreWithVersion`.
 - [#1116](https://github.com/crypto-org-chain/cronos/pull/1116) versiondb commands support sdk47 app hash calculation.
 
 ### Improvements

default.nix

@@ -49,7 +49,7 @@ buildGoApplication rec {
     else "-lrocksdb -pthread -lstdc++ -ldl";
 
   postFixup = lib.optionalString stdenv.isDarwin ''
-    ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.7.dylib" "${rocksdb}/lib/librocksdb.dylib" $out/bin/cronosd
+    ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.8.dylib" "${rocksdb}/lib/librocksdb.dylib" $out/bin/cronosd
   '';
 
   doCheck = false;

go.mod

@@ -6,7 +6,7 @@ require (
 	cosmossdk.io/errors v1.0.0-beta.7
 	cosmossdk.io/math v1.0.0-rc.0
 	github.com/armon/go-metrics v0.4.1
-	github.com/cosmos/cosmos-sdk v0.46.14
+	github.com/cosmos/cosmos-sdk v0.46.15-0.20230807104542-537257060180
 	github.com/cosmos/gogoproto v1.4.8
 	github.com/cosmos/ibc-go/v6 v6.2.0
 	github.com/crypto-org-chain/cronos/store v0.0.2

go.sum

@@ -408,8 +408,10 @@ github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk=
 github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0=
 github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE=
-github.com/cosmos/cosmos-sdk v0.46.14 h1:xlrUaMZT6QACdtWputs+ZxbAMWGYktWK+zlc8J4tKoE=
-github.com/cosmos/cosmos-sdk v0.46.14/go.mod h1:9MRixWsgoJ2UmVsCRRePtENFPP3cM+gTC5azEpxgllo=
+github.com/cosmos/cosmos-sdk v0.46.15-0.20230807090309-16d52690792e h1:vxDKg/iXN/qJHt/iqdCzGdTOU7lhn+1LEdloLSrZKFo=
+github.com/cosmos/cosmos-sdk v0.46.15-0.20230807090309-16d52690792e/go.mod h1:9MRixWsgoJ2UmVsCRRePtENFPP3cM+gTC5azEpxgllo=
+github.com/cosmos/cosmos-sdk v0.46.15-0.20230807104542-537257060180 h1:JgSAZEKce5UOGOykTVEcgJcUAL5kvpV0+RMfDnOna78=
+github.com/cosmos/cosmos-sdk v0.46.15-0.20230807104542-537257060180/go.mod h1:9MRixWsgoJ2UmVsCRRePtENFPP3cM+gTC5azEpxgllo=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=

gomod2nix.toml

@@ -131,8 +131,8 @@ schema = 3
     version = "v1.0.0-beta.1"
     hash = "sha256-oATkuj+fM5eBn+ywO+w/tL0AFSIEkx0J3Yz+VhVe0QA="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.14"
-    hash = "sha256-+NuFpghAkWRgitbS9ufKuobxKnjLIfTjrkTB1KMDrRw="
+    version = "v0.46.15-0.20230807104542-537257060180"
+    hash = "sha256-CKO0tUaz+ewDovdRuWvI40xYil7jAnsS823O4nZn6rA="
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
     hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="

integration_tests/cosmoscli.py

@@ -124,8 +124,22 @@ def init(cls, moniker, data_dir, node_rpc, cmd, chain_id):
         )
         return cls(data_dir, node_rpc, cmd)
 
-    def validate_genesis(self):
-        return self.raw("validate-genesis", home=self.data_dir)
+    def migrate_sdk_genesis(self, version, path):
+        return json.loads(self.raw("migrate", version, path))
+
+    def migrate_cronos_genesis(self, version, path):
+        return json.loads(
+            self.raw(
+                "tx",
+                "cronos",
+                "migrate",
+                version,
+                path,
+            )
+        )
+
+    def validate_genesis(self, path):
+        return self.raw("validate-genesis", path)
 
     def add_genesis_account(self, addr, coins, **kwargs):
         return self.raw(

integration_tests/test_basic.py

@@ -197,7 +197,12 @@ def test_statesync(cronos):
             "json-rpc": {
                 "address": "127.0.0.1:{EVMRPC_PORT}",
                 "ws-address": "127.0.0.1:{EVMRPC_PORT_WS}",
-            }
+            },
+            "memiavl": {
+                "enable": True,
+                "zero-copy": True,
+                "snapshot-interval": 5,
+            },
         },
     )
     clustercli.supervisor.startProcess(f"{clustercli.chain_id}-node{i}")

integration_tests/test_upgrade.py

@@ -80,13 +80,24 @@ def custom_cronos(tmp_path_factory):
     )
 
 
-def test_cosmovisor_upgrade(custom_cronos: Cronos):
+def test_cosmovisor_upgrade(custom_cronos: Cronos, tmp_path_factory):
     """
     - propose an upgrade and pass it
     - wait for it to happen
     - it should work transparently
     """
     cli = custom_cronos.cosmos_cli()
+    # export genesis from cronos v0.8.x
+    custom_cronos.supervisorctl("stop", "all")
+    migrate = tmp_path_factory.mktemp("migrate")
+    file_path0 = Path(migrate / "v0.8.json")
+    with open(file_path0, "w") as fp:
+        json.dump(json.loads(cli.export()), fp)
+        fp.flush()
+
+    custom_cronos.supervisorctl("start", "cronos_777-1-node0", "cronos_777-1-node1")
+    wait_for_port(ports.evmrpc_port(custom_cronos.base_port(0)))
+
     height = cli.block_height()
     target_height = height + 15
     print("upgrade height", target_height)
@@ -178,3 +189,18 @@ def test_cosmovisor_upgrade(custom_cronos: Cronos):
             "observe_ethereum_height_period": "50",
         }
     }
+
+    # migrate to sdk v0.46
+    custom_cronos.supervisorctl("stop", "all")
+    sdk_version = "v0.46"
+    file_path1 = Path(migrate / f"{sdk_version}.json")
+    with open(file_path1, "w") as fp:
+        json.dump(cli.migrate_sdk_genesis(sdk_version, str(file_path0)), fp)
+        fp.flush()
+    # migrate to cronos v1.0.x
+    cronos_version = "v1.0"
+    file_path2 = Path(migrate / f"{cronos_version}.json")
+    with open(file_path2, "w") as fp:
+        json.dump(cli.migrate_cronos_genesis(cronos_version, str(file_path1)), fp)
+        fp.flush()
+    print(cli.validate_genesis(str(file_path2)))

memiavl/export.go

@@ -63,14 +63,14 @@ func (db *DB) Snapshot(height uint64, protoWriter protoio.Writer) (returnErr err
 		if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{
 			Item: &snapshottypes.SnapshotItem_Store{
 				Store: &snapshottypes.SnapshotStoreItem{
-					Name: tree.name,
+					Name: tree.Name,
 				},
 			},
 		}); err != nil {
 			return err
 		}
 
-		exporter := tree.tree.Export()
+		exporter := tree.Tree.Export()
 		for {
 			node, err := exporter.Next()
 			if err == iavl.ExportDone {

memiavl/go.mod

@@ -20,7 +20,6 @@ require (
 	github.com/zbiljic/go-filelock v0.0.0-20170914061330-1dbf7103ab7d
 	golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.7.0
 )
 
 require (
@@ -69,6 +68,7 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	google.golang.org/grpc v1.54.0 // indirect

memiavl/import.go

@@ -20,13 +20,22 @@ import (
 // Import restore memiavl db from state-sync snapshot stream
 func Import(
 	dir string, height uint64, format uint32, protoReader protoio.Reader,
-) (snapshottypes.SnapshotItem, error) {
+) (item snapshottypes.SnapshotItem, err error) {
 	if height > math.MaxUint32 {
 		return snapshottypes.SnapshotItem{}, fmt.Errorf("version overflows uint32: %d", height)
 	}
 	snapshotDir := snapshotName(int64(height))
 	tmpDir := snapshotDir + "-tmp"
 
+	var fileLock FileLock
+	fileLock, err = LockFile(filepath.Join(dir, LockFileName))
+	if err != nil {
+		return snapshottypes.SnapshotItem{}, fmt.Errorf("fail to lock db: %w", err)
+	}
+	defer func() {
+		err = stderrors.Join(err, fileLock.Unlock())
+	}()
+
 	// Import nodes into stores. The first item is expected to be a SnapshotItem containing
 	// a SnapshotStoreItem, telling us which store to import into. The following items will contain
 	// SnapshotNodeItem (i.e. ExportNode) until we reach the next SnapshotStoreItem or EOF.

memiavl/multitree.go

@@ -19,9 +19,9 @@ import (
 
 const MetadataFileName = "__metadata"
 
-type namedTree struct {
-	tree *Tree
-	name string
+type NamedTree struct {
+	*Tree
+	Name string
 }
 
 // MultiTree manages multiple memiavl tree together,
@@ -47,8 +47,8 @@ type MultiTree struct {
 	zeroCopy  bool
 	cacheSize int
 
-	trees          []namedTree
-	treesByName    map[string]int // reversed index of the trees
+	trees          []NamedTree    // always ordered by tree name
+	treesByName    map[string]int // index of the trees by name
 	lastCommitInfo storetypes.CommitInfo
 
 	// the initial metadata loaded from disk snapshot
@@ -92,11 +92,11 @@ func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error)
 
 	sort.Strings(treeNames)
 
-	trees := make([]namedTree, len(treeNames))
+	trees := make([]NamedTree, len(treeNames))
 	treesByName := make(map[string]int, len(trees))
 	for i, name := range treeNames {
 		tree := treeMap[name]
-		trees[i] = namedTree{tree: tree, name: name}
+		trees[i] = NamedTree{Tree: tree, Name: name}
 		treesByName[name] = i
 	}
 
@@ -116,11 +116,16 @@ func LoadMultiTree(dir string, zeroCopy bool, cacheSize int) (*MultiTree, error)
 // TreeByName returns the tree by name, returns nil if not found
 func (t *MultiTree) TreeByName(name string) *Tree {
 	if i, ok := t.treesByName[name]; ok {
-		return t.trees[i].tree
+		return t.trees[i].Tree
 	}
 	return nil
 }
 
+// Trees returns all the trees together with the name, ordered by name.
+func (t *MultiTree) Trees() []NamedTree {
+	return t.trees
+}
+
 func (t *MultiTree) SetInitialVersion(initialVersion int64) error {
 	if initialVersion >= math.MaxUint32 {
 		return fmt.Errorf("version overflows uint32: %d", initialVersion)
@@ -131,8 +136,8 @@ func (t *MultiTree) SetInitialVersion(initialVersion int64) error {
 	}
 
 	for _, entry := range t.trees {
-		if !entry.tree.IsEmpty() {
-			return fmt.Errorf("tree is not empty: %s", entry.name)
+		if !entry.Tree.IsEmpty() {
+			return fmt.Errorf("tree is not empty: %s", entry.Name)
 		}
 	}
 
@@ -143,25 +148,25 @@ func (t *MultiTree) SetInitialVersion(initialVersion int64) error {
 func (t *MultiTree) setInitialVersion(initialVersion int64) {
 	t.initialVersion = uint32(initialVersion)
 	for _, entry := range t.trees {
-		entry.tree.initialVersion = t.initialVersion
+		entry.Tree.initialVersion = t.initialVersion
 	}
 }
 
 func (t *MultiTree) SetZeroCopy(zeroCopy bool) {
 	t.zeroCopy = zeroCopy
 	for _, entry := range t.trees {
-		entry.tree.SetZeroCopy(zeroCopy)
+		entry.Tree.SetZeroCopy(zeroCopy)
 	}
 }
 
 // Copy returns a snapshot of the tree which won't be corrupted by further modifications on the main tree.
 func (t *MultiTree) Copy(cacheSize int) *MultiTree {
-	trees := make([]namedTree, len(t.trees))
+	trees := make([]NamedTree, len(t.trees))
 	treesByName := make(map[string]int, len(t.trees))
 	for i, entry := range t.trees {
-		tree := entry.tree.Copy(cacheSize)
-		trees[i] = namedTree{tree: tree, name: entry.name}
-		treesByName[entry.name] = i
+		tree := entry.Tree.Copy(cacheSize)
+		trees[i] = NamedTree{Tree: tree, Name: entry.Name}
+		treesByName[entry.Name] = i
 	}
 
 	clone := *t
@@ -197,8 +202,8 @@ func (t *MultiTree) ApplyUpgrades(upgrades []*TreeNameUpgrade) error {
 	for _, upgrade := range upgrades {
 		switch {
 		case upgrade.Delete:
-			i := slices.IndexFunc(t.trees, func(entry namedTree) bool {
-				return entry.name == upgrade.Name
+			i := slices.IndexFunc(t.trees, func(entry NamedTree) bool {
+				return entry.Name == upgrade.Name
 			})
 			if i < 0 {
 				return fmt.Errorf("unknown tree name %s", upgrade.Name)
@@ -208,29 +213,29 @@ func (t *MultiTree) ApplyUpgrades(upgrades []*TreeNameUpgrade) error {
 			t.trees = t.trees[:len(t.trees)-1]
 		case upgrade.RenameFrom != "":
 			// rename tree
-			i := slices.IndexFunc(t.trees, func(entry namedTree) bool {
-				return entry.name == upgrade.RenameFrom
+			i := slices.IndexFunc(t.trees, func(entry NamedTree) bool {
+				return entry.Name == upgrade.RenameFrom
 			})
 			if i < 0 {
 				return fmt.Errorf("unknown tree name %s", upgrade.RenameFrom)
 			}
-			t.trees[i].name = upgrade.Name
+			t.trees[i].Name = upgrade.Name
 		default:
 			// add tree
 			tree := NewWithInitialVersion(uint32(nextVersion(t.Version(), t.initialVersion)), t.cacheSize)
-			t.trees = append(t.trees, namedTree{tree: tree, name: upgrade.Name})
+			t.trees = append(t.trees, NamedTree{Tree: tree, Name: upgrade.Name})
 		}
 	}
 
 	sort.SliceStable(t.trees, func(i, j int) bool {
-		return t.trees[i].name < t.trees[j].name
+		return t.trees[i].Name < t.trees[j].Name
 	})
 	t.treesByName = make(map[string]int, len(t.trees))
 	for i, tree := range t.trees {
-		if _, ok := t.treesByName[tree.name]; ok {
-			return fmt.Errorf("memiavl tree name conflicts: %s", tree.name)
+		if _, ok := t.treesByName[tree.Name]; ok {
+			return fmt.Errorf("memiavl tree name conflicts: %s", tree.Name)
 		}
-		t.treesByName[tree.name] = i
+		t.treesByName[tree.Name] = i
 	}
 
 	return nil
@@ -242,7 +247,7 @@ func (t *MultiTree) ApplyChangeSet(changeSets []*NamedChangeSet, updateCommitInf
 	version := nextVersion(t.lastCommitInfo.Version, t.initialVersion)
 
 	for _, cs := range changeSets {
-		tree := t.trees[t.treesByName[cs.Name]].tree
+		tree := t.trees[t.treesByName[cs.Name]].Tree
 
 		_, v, err := tree.ApplyChangeSet(cs.Changeset, updateCommitInfo)
 		if err != nil {
@@ -271,10 +276,10 @@ func (t *MultiTree) UpdateCommitInfo() []byte {
 	var infos []storetypes.StoreInfo
 	for _, entry := range t.trees {
 		infos = append(infos, storetypes.StoreInfo{
-			Name: entry.name,
+			Name: entry.Name,
 			CommitId: storetypes.CommitID{
-				Version: entry.tree.Version(),
-				Hash:    entry.tree.RootHash(),
+				Version: entry.Tree.Version(),
+				Hash:    entry.Tree.RootHash(),
 			},
 		})
 	}
@@ -337,7 +342,7 @@ func (t *MultiTree) WriteSnapshot(dir string) error {
 	// write the snapshots in parallel
 	g, _ := errgroup.WithContext(context.Background())
 	for _, entry := range t.trees {
-		tree, name := entry.tree, entry.name // https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
+		tree, name := entry.Tree, entry.Name // https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
 		g.Go(func() error {
 			return tree.WriteSnapshot(filepath.Join(dir, name))
 		})
@@ -377,7 +382,7 @@ func WriteFileSync(name string, data []byte) error {
 func (t *MultiTree) Close() error {
 	errs := make([]error, 0, len(t.trees))
 	for _, entry := range t.trees {
-		errs = append(errs, entry.tree.Close())
+		errs = append(errs, entry.Tree.Close())
 	}
 	t.trees = nil
 	t.treesByName = nil