diff --git a/node/Cargo.lock b/node/Cargo.lock index 346e2af7..222483f7 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -62,7 +62,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -88,9 +88,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -103,43 +103,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "assert_matches" @@ -161,9 +161,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -172,24 +172,24 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -200,15 +200,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.8.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -218,11 +218,11 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.20.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" dependencies = [ - "bindgen 0.69.4", + "bindgen 0.69.5", "cc", "cmake", "dunce", @@ -244,17 +244,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -308,14 +308,14 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -330,7 +330,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.72", + "syn 2.0.87", "which", ] @@ -411,9 +411,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytesize" @@ -440,12 +440,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.7" +version = "1.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -554,9 +555,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.13" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -564,9 +565,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.13" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -576,14 +577,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -594,18 +595,18 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -650,15 +651,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -760,7 +761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -796,7 +797,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -820,7 +821,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -831,7 +832,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -882,6 +883,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "dtoa" version = "1.0.9" @@ -890,9 +902,9 @@ checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" @@ -986,7 +998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1012,9 +1024,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "ff" @@ -1061,9 +1073,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1075,9 +1087,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1085,44 +1097,44 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1170,9 +1182,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -1193,9 +1205,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1230,6 +1242,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + [[package]] name = "heck" version = "0.5.0" @@ -1242,6 +1260,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -1263,7 +1287,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1324,9 +1348,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1342,9 +1366,9 @@ checksum = "f58b778a5761513caf593693f8951c97a5b610841e754788400f32102eefdff1" [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ "bytes", "futures-channel", @@ -1365,9 +1389,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -1386,17 +1410,17 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "log", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1405,11 +1429,11 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "pin-project-lite", "tokio", @@ -1418,24 +1442,141 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.0", "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1444,12 +1585,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1468,12 +1620,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.1", ] [[package]] @@ -1496,13 +1648,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1566,9 +1718,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1646,7 +1798,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls", "hyper-util", "jsonrpsee-core", @@ -1673,7 +1825,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "jsonrpsee-core", "jsonrpsee-types", @@ -1705,9 +1857,9 @@ dependencies = [ [[package]] name = "k256" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", "ecdsa", @@ -1767,7 +1919,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls", "hyper-timeout", "hyper-util", @@ -1816,7 +1968,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1832,7 +1984,7 @@ dependencies = [ "backoff", "derivative", "futures", - "hashbrown", + "hashbrown 0.14.5", "json-patch", "k8s-openapi", "kube-client", @@ -1861,9 +2013,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libloading" @@ -1893,9 +2045,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "pkg-config", @@ -1908,6 +2060,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -1944,7 +2102,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1958,9 +2116,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.10.0" +version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", @@ -2001,7 +2159,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2018,23 +2176,23 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2109,24 +2267,24 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "object" -version = "0.36.2" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -2163,9 +2321,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -2220,9 +2378,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -2231,9 +2389,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" dependencies = [ "pest", "pest_generator", @@ -2241,22 +2399,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" dependencies = [ "once_cell", "pest", @@ -2275,29 +2433,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -2317,15 +2475,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -2336,15 +2494,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -2380,18 +2538,18 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy 0.6.6", + "zerocopy", ] [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -2399,19 +2557,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2436,7 +2594,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2466,7 +2624,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.72", + "syn 2.0.87", "tempfile", ] @@ -2480,7 +2638,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2546,9 +2704,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -2614,23 +2772,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -2644,13 +2802,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -2661,9 +2819,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rfc6979" @@ -2687,7 +2845,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2720,31 +2878,31 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "aws-lc-rs", "log", @@ -2758,9 +2916,22 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -2771,25 +2942,24 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-platform-verifier" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93bda3f493b9abe5b93b3e7e3ecde0df292f2bd28c0296b90586ee0055ff5123" +checksum = "afbb878bdfdf63a336a5e63561b1835e7a8c91524f51621db870169eac84b490" dependencies = [ "core-foundation", "core-foundation-sys", @@ -2797,7 +2967,7 @@ dependencies = [ "log", "once_cell", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki", "security-framework", @@ -2814,9 +2984,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "aws-lc-rs", "ring", @@ -2841,11 +3011,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -2869,7 +3039,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2918,9 +3088,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -2934,9 +3104,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] @@ -2953,13 +3123,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2970,14 +3140,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.122" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -3112,7 +3282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -3178,25 +3348,37 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -3216,12 +3398,12 @@ checksum = "f9b53c7124dd88026d5d98a1eb1fd062a578b7d783017c9298825526c7fb6427" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "tester" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "clap", @@ -3235,22 +3417,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3292,35 +3474,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] -name = "tinytemplate" -version = "1.2.1" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "serde", - "serde_json", + "displaydoc", + "zerovec", ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "tinytemplate" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ - "tinyvec_macros", + "serde", + "serde_json", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tls-listener" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a296135fdab7b3a1f708c338c50bab570bcd77d44080cde9341df45c0c6d73" +checksum = "0f1d8809f604e448c7bc53a5a0e4c2a0a20ba44cb1fb407314c8eeccb92127f9" dependencies = [ "futures-util", "pin-project-lite", @@ -3331,9 +3508,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.39.2" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -3344,7 +3521,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -3355,7 +3532,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3371,9 +3548,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -3383,9 +3560,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -3434,15 +3611,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -3464,7 +3641,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3520,36 +3697,21 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - -[[package]] -name = "unicode-bidi" -version = "0.3.15" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.23" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "universal-hash" @@ -3575,15 +3737,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -3628,7 +3802,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "671d3b894d5d0849f0a597f56bf071f42d4f2a1cbcf2f78ca21f870ab7c0cc2b" dependencies = [ - "hyper 0.14.30", + "hyper 0.14.31", "once_cell", "tokio", "tracing", @@ -3643,7 +3817,7 @@ checksum = "6a511871dc5de990a3b2a0e715facfbc5da848c0c0395597a1415029fb7c250a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3673,34 +3847,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3708,28 +3883,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -3737,9 +3912,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -3774,11 +3949,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -3796,6 +3971,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -3860,20 +4044,46 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] -name = "zerocopy" -version = "0.6.6" +name = "yoke" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" dependencies = [ - "byteorder", - "zerocopy-derive 0.6.6", + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -3882,29 +4092,40 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive 0.7.35", + "byteorder", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] -name = "zerocopy-derive" -version = "0.7.35" +name = "zerofrom" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -3924,12 +4145,34 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] name = "zksync_concurrency" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "assert_matches", @@ -3947,7 +4190,7 @@ dependencies = [ [[package]] name = "zksync_consensus_bft" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "assert_matches", @@ -3971,7 +4214,7 @@ dependencies = [ [[package]] name = "zksync_consensus_crypto" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "blst", @@ -3991,7 +4234,7 @@ dependencies = [ [[package]] name = "zksync_consensus_executor" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "async-trait", @@ -4013,7 +4256,7 @@ dependencies = [ [[package]] name = "zksync_consensus_network" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "assert_matches", @@ -4023,7 +4266,7 @@ dependencies = [ "bytesize", "http-body-util", "human-repr", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "im", "once_cell", @@ -4051,7 +4294,7 @@ dependencies = [ [[package]] name = "zksync_consensus_roles" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "assert_matches", @@ -4072,7 +4315,7 @@ dependencies = [ [[package]] name = "zksync_consensus_storage" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "assert_matches", @@ -4094,7 +4337,7 @@ dependencies = [ [[package]] name = "zksync_consensus_tools" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "async-trait", @@ -4129,7 +4372,7 @@ dependencies = [ [[package]] name = "zksync_consensus_utils" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "rand", @@ -4139,7 +4382,7 @@ dependencies = [ [[package]] name = "zksync_protobuf" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "bit-vec", @@ -4161,7 +4404,7 @@ dependencies = [ [[package]] name = "zksync_protobuf_build" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "heck", @@ -4171,14 +4414,14 @@ dependencies = [ "prost-reflect", "protox", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/node/actors/bft/src/replica/block.rs b/node/actors/bft/src/chonky_bft/block.rs similarity index 54% rename from node/actors/bft/src/replica/block.rs rename to node/actors/bft/src/chonky_bft/block.rs index 2d6dc0a8..3ec22860 100644 --- a/node/actors/bft/src/replica/block.rs +++ b/node/actors/bft/src/chonky_bft/block.rs @@ -1,31 +1,17 @@ use super::StateMachine; -use zksync_concurrency::ctx; +use zksync_concurrency::{ctx, error::Wrap as _}; use zksync_consensus_roles::validator; +use zksync_consensus_storage as storage; impl StateMachine { /// Tries to build a finalized block from the given CommitQC. We simply search our /// block proposal cache for the matching block, and if we find it we build the block. - /// If this method succeeds, it sends the finalized block to the executor. - /// It also updates the High QC in the replica state machine, if the received QC is - /// higher. - #[tracing::instrument(level = "debug", skip_all)] + /// If this method succeeds, it saves the finalized block to storage. pub(crate) async fn save_block( &mut self, ctx: &ctx::Ctx, commit_qc: &validator::CommitQC, ) -> ctx::Result<()> { - // Update high_qc. - if self - .high_qc - .as_ref() - .map(|qc| qc.view().number < commit_qc.view().number) - .unwrap_or(true) - { - self.high_qc = Some(commit_qc.clone()); - } - // TODO(gprusak): for availability of finalized blocks, - // replicas should be able to broadcast highest quorums without - // the corresponding block (same goes for synchronization). let Some(cache) = self.block_proposal_cache.get(&commit_qc.header().number) else { return Ok(()); }; @@ -46,7 +32,11 @@ impl StateMachine { .block_store .queue_block(ctx, block.clone().into()) .await?; + // For availability, replica should not proceed until it stores the block persistently. + // Rationale is that after save_block, there is start_new_view which prunes the + // cache. Without persisting this block, if all replicas crash just after + // start_new_view, the payload becomes unavailable. self.config .block_store .wait_until_persisted(ctx, block.header().number) @@ -55,6 +45,32 @@ impl StateMachine { let number_metric = &crate::metrics::METRICS.finalized_block_number; let current_number = number_metric.get(); number_metric.set(current_number.max(block.header().number.0)); + + Ok(()) + } + + /// Backups the replica state to DB. + pub(crate) async fn backup_state(&self, ctx: &ctx::Ctx) -> ctx::Result<()> { + let mut proposals = vec![]; + for (number, payloads) in &self.block_proposal_cache { + proposals.extend(payloads.values().map(|p| storage::Proposal { + number: *number, + payload: p.clone(), + })); + } + let backup = storage::ReplicaState { + view: self.view_number, + phase: self.phase, + high_vote: self.high_vote.clone(), + high_commit_qc: self.high_commit_qc.clone(), + high_timeout_qc: self.high_timeout_qc.clone(), + proposals, + }; + self.config + .replica_store + .set_state(ctx, &backup) + .await + .wrap("set_state()")?; Ok(()) } } diff --git a/node/actors/bft/src/chonky_bft/commit.rs b/node/actors/bft/src/chonky_bft/commit.rs new file mode 100644 index 00000000..f824f267 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/commit.rs @@ -0,0 +1,161 @@ +use super::StateMachine; +use crate::metrics; +use std::collections::HashSet; +use zksync_concurrency::{ctx, error::Wrap, metrics::LatencyHistogramExt as _}; +use zksync_consensus_roles::validator; + +/// Errors that can occur when processing a ReplicaCommit message. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + /// Message signer isn't part of the validator set. + #[error("message signer isn't part of the validator set (signer: {signer:?})")] + NonValidatorSigner { + /// Signer of the message. + signer: Box, + }, + /// Past view. + #[error("past view (current view: {current_view:?})")] + Old { + /// Current view. + current_view: validator::ViewNumber, + }, + /// Duplicate signer. We already have a commit message from the same validator + /// for the same or past view. + #[error("duplicate signer (message view: {message_view:?}, signer: {signer:?})")] + DuplicateSigner { + /// View number of the message. + message_view: validator::ViewNumber, + /// Signer of the message. + signer: Box, + }, + /// Invalid message signature. + #[error("invalid signature: {0:#}")] + InvalidSignature(#[source] anyhow::Error), + /// Invalid message. + #[error("invalid message: {0:#}")] + InvalidMessage(#[source] validator::ReplicaCommitVerifyError), + /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. + #[error(transparent)] + Internal(#[from] ctx::Error), +} + +impl Wrap for Error { + fn with_wrap C>( + self, + f: F, + ) -> Self { + match self { + Error::Internal(err) => Error::Internal(err.with_wrap(f)), + err => err, + } + } +} + +impl StateMachine { + /// Processes a ReplicaCommit message. + pub(crate) async fn on_commit( + &mut self, + ctx: &ctx::Ctx, + signed_message: validator::Signed, + ) -> Result<(), Error> { + // ----------- Checking origin of the message -------------- + + // Unwrap message. + let message = &signed_message.msg; + let author = &signed_message.key; + + // Check that the message signer is in the validator committee. + if !self.config.genesis().validators.contains(author) { + return Err(Error::NonValidatorSigner { + signer: author.clone().into(), + }); + } + + // If the message is from a past view, ignore it. + if message.view.number < self.view_number { + return Err(Error::Old { + current_view: self.view_number, + }); + } + + // If we already have a message from the same validator for the same or past view, ignore it. + if let Some(&view) = self.commit_views_cache.get(author) { + if view >= message.view.number { + return Err(Error::DuplicateSigner { + message_view: message.view.number, + signer: author.clone().into(), + }); + } + } + + // ----------- Checking the signed part of the message -------------- + + // Check the signature on the message. + signed_message.verify().map_err(Error::InvalidSignature)?; + + message + .verify(self.config.genesis()) + .map_err(Error::InvalidMessage)?; + + // ----------- All checks finished. Now we process the message. -------------- + + // We add the message to the incrementally-constructed QC. + let commit_qc = self + .commit_qcs_cache + .entry(message.view.number) + .or_default() + .entry(message.clone()) + .or_insert_with(|| validator::CommitQC::new(message.clone(), self.config.genesis())); + + // Should always succeed as all checks have been already performed + commit_qc + .add(&signed_message, self.config.genesis()) + .expect("could not add message to CommitQC"); + + // Calculate the CommitQC signers weight. + let weight = self.config.genesis().validators.weight(&commit_qc.signers); + + // Update view number of last commit message for author + self.commit_views_cache + .insert(author.clone(), message.view.number); + + // Clean up commit_qcs for the case that no replica is at the view + // of a given CommitQC. + // This prevents commit_qcs map from growing indefinitely in case some + // malicious replica starts spamming messages for future views. + let active_views: HashSet<_> = self.commit_views_cache.values().collect(); + self.commit_qcs_cache + .retain(|view_number, _| active_views.contains(view_number)); + + // Now we check if we have enough weight to continue. If not, we wait for more messages. + if weight < self.config.genesis().validators.quorum_threshold() { + return Ok(()); + }; + + // ----------- We have a QC. Now we process it. -------------- + + // Consume the created commit QC for this view. + let commit_qc = self + .commit_qcs_cache + .remove(&message.view.number) + .unwrap() + .remove(message) + .unwrap(); + + // We update our state with the new commit QC. + self.process_commit_qc(ctx, &commit_qc) + .await + .wrap("process_commit_qc()")?; + + // Metrics. We observe the latency of committing to a block measured + // from the start of this view. + metrics::METRICS + .commit_latency + .observe_latency(ctx.now() - self.view_start); + + // Start a new view. + self.start_new_view(ctx, message.view.number.next()).await?; + + Ok(()) + } +} diff --git a/node/actors/bft/src/chonky_bft/mod.rs b/node/actors/bft/src/chonky_bft/mod.rs new file mode 100644 index 00000000..9c8e7790 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/mod.rs @@ -0,0 +1,315 @@ +use crate::{io::OutputMessage, metrics, Config}; +use std::{ + cmp::max, + collections::{BTreeMap, HashMap}, + sync::Arc, +}; +use zksync_concurrency::{ + ctx, + error::Wrap as _, + metrics::LatencyHistogramExt as _, + sync::{self, prunable_mpsc::SelectionFunctionResult}, + time, +}; +use zksync_consensus_network::io::ConsensusReq; +use zksync_consensus_roles::validator::{self, ConsensusMsg}; + +mod block; +mod commit; +mod new_view; +mod proposal; +/// The proposer module contains the logic for the proposer role in ChonkyBFT. +pub(crate) mod proposer; +#[cfg(test)] +pub(crate) mod testonly; +#[cfg(test)] +mod tests; +mod timeout; + +/// The duration of the view timeout. +pub(crate) const VIEW_TIMEOUT_DURATION: time::Duration = time::Duration::milliseconds(2000); + +/// The StateMachine struct contains the state of the replica and implements all the +/// logic of ChonkyBFT. +#[derive(Debug)] +pub(crate) struct StateMachine { + /// Consensus configuration. + pub(crate) config: Arc, + /// Pipe through which replica sends network messages. + pub(super) outbound_pipe: ctx::channel::UnboundedSender, + /// Pipe through which replica receives network requests. + pub(crate) inbound_pipe: sync::prunable_mpsc::Receiver, + /// The sender part of the proposer watch channel. This is used to notify the proposer loop + /// and send the needed justification. + pub(crate) proposer_pipe: sync::watch::Sender>, + + /// The current view number. + pub(crate) view_number: validator::ViewNumber, + /// The current phase. + pub(crate) phase: validator::Phase, + /// The highest block proposal that the replica has committed to. + pub(crate) high_vote: Option, + /// The highest commit quorum certificate known to the replica. + pub(crate) high_commit_qc: Option, + /// The highest timeout quorum certificate known to the replica. + pub(crate) high_timeout_qc: Option, + + /// A cache of the received block proposals. + pub(crate) block_proposal_cache: + BTreeMap>, + /// Latest view each validator has signed a ReplicaCommit message for. + pub(crate) commit_views_cache: BTreeMap, + /// Commit QCs indexed by view number and then by message. + pub(crate) commit_qcs_cache: + BTreeMap>, + /// Latest view each validator has signed a ReplicaTimeout message for. + pub(crate) timeout_views_cache: BTreeMap, + /// Timeout QCs indexed by view number. + pub(crate) timeout_qcs_cache: BTreeMap, + + /// The deadline to receive a proposal for this view before timing out. + pub(crate) view_timeout: time::Deadline, + /// Time when the current view phase has started. Used for metrics. + pub(crate) view_start: time::Instant, +} + +impl StateMachine { + /// Creates a new [`StateMachine`] instance, attempting to recover a past state from the storage module, + /// otherwise initializes the state machine with the current head block. + /// + /// Returns a tuple containing: + /// * The newly created [`StateMachine`] instance. + /// * A sender handle that should be used to send values to be processed by the instance, asynchronously. + pub(crate) async fn start( + ctx: &ctx::Ctx, + config: Arc, + outbound_pipe: ctx::channel::UnboundedSender, + proposer_pipe: sync::watch::Sender>, + ) -> ctx::Result<(Self, sync::prunable_mpsc::Sender)> { + let backup = config.replica_store.state(ctx).await?; + + let mut block_proposal_cache: BTreeMap<_, HashMap<_, _>> = BTreeMap::new(); + for proposal in backup.proposals { + block_proposal_cache + .entry(proposal.number) + .or_default() + .insert(proposal.payload.hash(), proposal.payload); + } + + let (send, recv) = sync::prunable_mpsc::channel( + StateMachine::inbound_filter_predicate, + StateMachine::inbound_selection_function, + ); + + let this = Self { + config, + outbound_pipe, + inbound_pipe: recv, + proposer_pipe, + view_number: backup.view, + phase: backup.phase, + high_vote: backup.high_vote, + high_commit_qc: backup.high_commit_qc, + high_timeout_qc: backup.high_timeout_qc, + block_proposal_cache, + commit_views_cache: BTreeMap::new(), + commit_qcs_cache: BTreeMap::new(), + timeout_views_cache: BTreeMap::new(), + timeout_qcs_cache: BTreeMap::new(), + view_timeout: time::Deadline::Finite(ctx.now() + VIEW_TIMEOUT_DURATION), + view_start: ctx.now(), + }; + + Ok((this, send)) + } + + /// Runs a loop to process incoming messages (may be `None` if the channel times out while waiting for a message). + /// This is the main entry point for the state machine, + /// potentially triggering state modifications and message sending to the executor. + pub(crate) async fn run(mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { + self.view_start = ctx.now(); + + // If this is the first view, we immediately timeout. This will force the replicas + // to synchronize right at the beginning and will provide a justification for the + // next view. This is necessary because the first view is not justified by any + // previous view. + if self.view_number == validator::ViewNumber(0) { + self.start_timeout(ctx).await?; + } + + // Main loop. + loop { + let recv = self + .inbound_pipe + .recv(&ctx.with_deadline(self.view_timeout)) + .await; + + // Check for non-timeout cancellation. + if !ctx.is_active() { + return Ok(()); + } + + // Check for timeout. If we are already in a timeout phase, we don't + // timeout again. Note though that the underlying network implementation + // needs to keep retrying messages until they are delivered. Otherwise + // the consensus can halt! + let Some(req) = recv.ok() else { + if self.phase != validator::Phase::Timeout { + self.start_timeout(ctx).await?; + } + continue; + }; + + // Process the message. + let now = ctx.now(); + let label = match &req.msg.msg { + ConsensusMsg::LeaderProposal(_) => { + let res = match self + .on_proposal(ctx, req.msg.cast().unwrap()) + .await + .wrap("on_proposal()") + { + Ok(()) => Ok(()), + Err(err) => { + match err { + // If the error is internal, we stop here. + proposal::Error::Internal(err) => { + tracing::error!("on_proposal: internal error: {err:#}"); + return Err(err); + } + // If the error is due to an old message, we log it at a lower level. + proposal::Error::Old { .. } => { + tracing::debug!("on_proposal: {err:#}"); + } + _ => { + tracing::warn!("on_proposal: {err:#}"); + } + } + Err(()) + } + }; + metrics::ConsensusMsgLabel::LeaderProposal.with_result(&res) + } + ConsensusMsg::ReplicaCommit(_) => { + let res = match self + .on_commit(ctx, req.msg.cast().unwrap()) + .await + .wrap("on_commit()") + { + Ok(()) => Ok(()), + Err(err) => { + match err { + // If the error is internal, we stop here. + commit::Error::Internal(err) => { + tracing::error!("on_commit: internal error: {err:#}"); + return Err(err); + } + // If the error is due to an old message, we log it at a lower level. + commit::Error::Old { .. } => { + tracing::debug!("on_commit: {err:#}"); + } + _ => { + tracing::warn!("on_commit: {err:#}"); + } + } + Err(()) + } + }; + metrics::ConsensusMsgLabel::ReplicaCommit.with_result(&res) + } + ConsensusMsg::ReplicaTimeout(_) => { + let res = match self + .on_timeout(ctx, req.msg.cast().unwrap()) + .await + .wrap("on_timeout()") + { + Ok(()) => Ok(()), + Err(err) => { + match err { + // If the error is internal, we stop here. + timeout::Error::Internal(err) => { + tracing::error!("on_timeout: internal error: {err:#}"); + return Err(err); + } + // If the error is due to an old message, we log it at a lower level. + timeout::Error::Old { .. } => { + tracing::debug!("on_timeout: {err:#}"); + } + _ => { + tracing::warn!("on_timeout: {err:#}"); + } + } + Err(()) + } + }; + metrics::ConsensusMsgLabel::ReplicaTimeout.with_result(&res) + } + ConsensusMsg::ReplicaNewView(_) => { + let res = match self + .on_new_view(ctx, req.msg.cast().unwrap()) + .await + .wrap("on_new_view()") + { + Ok(()) => Ok(()), + Err(err) => { + match err { + // If the error is internal, we stop here. + new_view::Error::Internal(err) => { + tracing::error!("on_new_view: internal error: {err:#}"); + return Err(err); + } + // If the error is due to an old message, we log it at a lower level. + new_view::Error::Old { .. } => { + tracing::debug!("on_new_view: {err:#}"); + } + _ => { + tracing::warn!("on_new_view: {err:#}"); + } + } + Err(()) + } + }; + metrics::ConsensusMsgLabel::ReplicaNewView.with_result(&res) + } + }; + metrics::METRICS.message_processing_latency[&label].observe_latency(ctx.now() - now); + + // Notify network actor that the message has been processed. + // Ignore sending error. + let _ = req.ack.send(()); + } + } + + fn inbound_filter_predicate(new_req: &ConsensusReq) -> bool { + // Verify message signature + new_req.msg.verify().is_ok() + } + + fn inbound_selection_function( + old_req: &ConsensusReq, + new_req: &ConsensusReq, + ) -> SelectionFunctionResult { + if old_req.msg.key != new_req.msg.key || old_req.msg.msg.label() != new_req.msg.msg.label() + { + SelectionFunctionResult::Keep + } else { + // Discard older message + if old_req.msg.msg.view().number < new_req.msg.msg.view().number { + SelectionFunctionResult::DiscardOld + } else { + SelectionFunctionResult::DiscardNew + } + } + } + + /// Processes a (already verified) CommitQC. It bumps the local high_commit_qc and if + /// we have the proposal corresponding to this qc, we save the corresponding block to DB. + pub(crate) async fn process_commit_qc( + &mut self, + ctx: &ctx::Ctx, + qc: &validator::CommitQC, + ) -> ctx::Result<()> { + self.high_commit_qc = max(Some(qc.clone()), self.high_commit_qc.clone()); + self.save_block(ctx, qc).await.wrap("save_block()") + } +} diff --git a/node/actors/bft/src/chonky_bft/new_view.rs b/node/actors/bft/src/chonky_bft/new_view.rs new file mode 100644 index 00000000..ff401ebb --- /dev/null +++ b/node/actors/bft/src/chonky_bft/new_view.rs @@ -0,0 +1,175 @@ +use std::cmp::max; + +use super::StateMachine; +use crate::{chonky_bft::VIEW_TIMEOUT_DURATION, metrics}; +use zksync_concurrency::{ctx, error::Wrap, metrics::LatencyHistogramExt as _, time}; +use zksync_consensus_network::io::ConsensusInputMessage; +use zksync_consensus_roles::validator; + +/// Errors that can occur when processing a ReplicaNewView message. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + /// Message signer isn't part of the validator set. + #[error("message signer isn't part of the validator set (signer: {signer:?})")] + NonValidatorSigner { + /// Signer of the message. + signer: Box, + }, + /// Past view or phase. + #[error("past view (current view: {current_view:?})")] + Old { + /// Current view. + current_view: validator::ViewNumber, + }, + /// Invalid message signature. + #[error("invalid signature: {0:#}")] + InvalidSignature(#[source] anyhow::Error), + /// Invalid message. + #[error("invalid message: {0:#}")] + InvalidMessage(#[source] validator::ReplicaNewViewVerifyError), + /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. + #[error(transparent)] + Internal(#[from] ctx::Error), +} + +impl Wrap for Error { + fn with_wrap C>( + self, + f: F, + ) -> Self { + match self { + Error::Internal(err) => Error::Internal(err.with_wrap(f)), + err => err, + } + } +} + +impl StateMachine { + /// Processes a ReplicaNewView message. + pub(crate) async fn on_new_view( + &mut self, + ctx: &ctx::Ctx, + signed_message: validator::Signed, + ) -> Result<(), Error> { + // ----------- Checking origin of the message -------------- + + // Unwrap message. + let message = &signed_message.msg; + let author = &signed_message.key; + + // Check that the message signer is in the validator committee. + if !self.config.genesis().validators.contains(author) { + return Err(Error::NonValidatorSigner { + signer: author.clone().into(), + }); + } + + // If the message is from a past view, ignore it. + if message.view().number < self.view_number { + return Err(Error::Old { + current_view: self.view_number, + }); + } + + // ----------- Checking the signed part of the message -------------- + + // Check the signature on the message. + signed_message.verify().map_err(Error::InvalidSignature)?; + + message + .verify(self.config.genesis()) + .map_err(Error::InvalidMessage)?; + + // ----------- All checks finished. Now we process the message. -------------- + + // Update the state machine. + match &message.justification { + validator::ProposalJustification::Commit(qc) => self + .process_commit_qc(ctx, qc) + .await + .wrap("process_commit_qc()")?, + validator::ProposalJustification::Timeout(qc) => { + if let Some(high_qc) = qc.high_qc() { + self.process_commit_qc(ctx, high_qc) + .await + .wrap("process_commit_qc()")?; + } + self.high_timeout_qc = max(Some(qc.clone()), self.high_timeout_qc.clone()); + } + }; + + // If the message is for a future view, we need to start a new view. + if message.view().number > self.view_number { + self.start_new_view(ctx, message.view().number).await?; + } + + Ok(()) + } + + /// This blocking method is used whenever we start a new view. + pub(crate) async fn start_new_view( + &mut self, + ctx: &ctx::Ctx, + view: validator::ViewNumber, + ) -> ctx::Result<()> { + // Update the state machine. + self.view_number = view; + self.phase = validator::Phase::Prepare; + self.proposer_pipe + .send(Some(self.get_justification())) + .expect("justification_watch.send() failed"); + + // Clear the block proposal cache. + if let Some(qc) = self.high_commit_qc.as_ref() { + self.block_proposal_cache + .retain(|k, _| k > &qc.header().number); + } + + // Backup our state. + self.backup_state(ctx).await.wrap("backup_state()")?; + + // Broadcast our new view message. + let output_message = ConsensusInputMessage { + message: self + .config + .secret_key + .sign_msg(validator::ConsensusMsg::ReplicaNewView( + validator::ReplicaNewView { + justification: self.get_justification(), + }, + )), + }; + self.outbound_pipe.send(output_message.into()); + + // Log the event and update the metrics. + tracing::info!("Starting view {}", self.view_number); + metrics::METRICS.replica_view_number.set(self.view_number.0); + let now = ctx.now(); + metrics::METRICS + .view_latency + .observe_latency(now - self.view_start); + self.view_start = now; + + // Reset the timeout. + self.view_timeout = time::Deadline::Finite(ctx.now() + VIEW_TIMEOUT_DURATION); + + Ok(()) + } + + /// Makes a justification (for a ReplicaNewView or a LeaderProposal) based on the current state. + pub(crate) fn get_justification(&self) -> validator::ProposalJustification { + // We need some QC in order to be able to create a justification. + // In fact, it should be impossible to get here without a QC. Because + // we only get here after starting a new view, which requires a QC. + assert!(self.high_commit_qc.is_some() || self.high_timeout_qc.is_some()); + + // We use the highest QC as the justification. If both have the same view, we use the CommitQC. + if self.high_commit_qc.as_ref().map(|x| x.view()) + >= self.high_timeout_qc.as_ref().map(|x| &x.view) + { + validator::ProposalJustification::Commit(self.high_commit_qc.clone().unwrap()) + } else { + validator::ProposalJustification::Timeout(self.high_timeout_qc.clone().unwrap()) + } + } +} diff --git a/node/actors/bft/src/chonky_bft/proposal.rs b/node/actors/bft/src/chonky_bft/proposal.rs new file mode 100644 index 00000000..b89f09d8 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/proposal.rs @@ -0,0 +1,257 @@ +use crate::metrics; + +use super::StateMachine; +use std::cmp::max; +use zksync_concurrency::{ctx, error::Wrap, metrics::LatencyHistogramExt as _}; +use zksync_consensus_network::io::ConsensusInputMessage; +use zksync_consensus_roles::validator::{self, BlockHeader, BlockNumber}; + +/// Errors that can occur when processing a LeaderProposal message. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + /// Message for a past view or phase. + #[error( + "message for a past view / phase (current view: {current_view:?}, current phase: {current_phase:?})" + )] + Old { + /// Current view. + current_view: validator::ViewNumber, + /// Current phase. + current_phase: validator::Phase, + }, + /// Invalid leader. + #[error( + "invalid leader (correct leader: {correct_leader:?}, received leader: {received_leader:?})" + )] + InvalidLeader { + /// Correct leader. + correct_leader: validator::PublicKey, + /// Received leader. + received_leader: validator::PublicKey, + }, + /// Invalid message signature. + #[error("invalid signature: {0:#}")] + InvalidSignature(#[source] anyhow::Error), + /// Invalid message. + #[error("invalid message: {0:#}")] + InvalidMessage(#[source] validator::LeaderProposalVerifyError), + /// Leader proposed a block that was already pruned from replica's storage. + #[error("leader proposed a block that was already pruned from replica's storage")] + ProposalAlreadyPruned, + /// Reproposal with an unnecessary payload. + #[error("reproposal with an unnecessary payload")] + ReproposalWithPayload, + /// Block proposal payload missing. + #[error("block proposal payload missing")] + MissingPayload, + /// Oversized payload. + #[error("block proposal with an oversized payload (payload size: {payload_size})")] + ProposalOversizedPayload { + /// Size of the payload. + payload_size: usize, + }, + /// Previous payload missing. + #[error("previous block proposal payload missing from store (block number: {prev_number})")] + MissingPreviousPayload { + /// The number of the missing block + prev_number: BlockNumber, + }, + /// Invalid payload. + #[error("invalid payload: {0:#}")] + InvalidPayload(#[source] anyhow::Error), + /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. + #[error(transparent)] + Internal(#[from] ctx::Error), +} + +impl Wrap for Error { + fn with_wrap C>( + self, + f: F, + ) -> Self { + match self { + Error::Internal(err) => Error::Internal(err.with_wrap(f)), + err => err, + } + } +} + +impl StateMachine { + /// Processes a LeaderProposal message. + pub(crate) async fn on_proposal( + &mut self, + ctx: &ctx::Ctx, + signed_message: validator::Signed, + ) -> Result<(), Error> { + // ----------- Checking origin of the message -------------- + + // Unwrap message. + let message = &signed_message.msg; + let author = &signed_message.key; + let view = message.view().number; + + // Check that the message is for the current view or a future view. We only allow proposals for + // the current view if we have not voted or timed out yet. + if view < self.view_number + || (view == self.view_number && self.phase != validator::Phase::Prepare) + { + return Err(Error::Old { + current_view: self.view_number, + current_phase: self.phase, + }); + } + + // Check that it comes from the correct leader. + let leader = self.config.genesis().view_leader(view); + if author != &leader { + return Err(Error::InvalidLeader { + correct_leader: leader, + received_leader: author.clone(), + }); + } + + // ----------- Checking the message -------------- + + signed_message.verify().map_err(Error::InvalidSignature)?; + + message + .verify(self.config.genesis()) + .map_err(Error::InvalidMessage)?; + + let (implied_block_number, implied_block_hash) = message + .justification + .get_implied_block(self.config.genesis()); + + // Replica MUSTN'T vote for blocks which have been already pruned for storage. + // (because it won't be able to persist and broadcast them once finalized). + // TODO(gprusak): it should never happen, we should add safety checks to prevent + // pruning blocks not known to be finalized. + if implied_block_number < self.config.block_store.queued().first { + return Err(Error::ProposalAlreadyPruned); + } + + let block_hash = match implied_block_hash { + // This is a reproposal. + // We let the leader repropose blocks without sending them in the proposal + // (it sends only the block number + block hash). That allows a leader to + // repropose a block without having it stored. Sending reproposals without + // a payload is an optimization that allows us to not wait for a leader that + // has the previous proposal stored (which can take 4f views), and to somewhat + // speed up reproposals by skipping block broadcast. + // This only saves time because we have a gossip network running in parallel, + // and any time a replica is able to create a finalized block (by possessing + // both the block and the commit QC) it broadcasts the finalized block (this + // was meant to propagate the block to full nodes, but of course validators + // will end up receiving it as well). + Some(hash) => { + // We check that the leader didn't send a payload with the reproposal. + // This isn't technically needed for the consensus to work (it will remain + // safe and live), but it's a good practice to avoid unnecessary data in + // blockchain. + // This unnecessary payload would also effectively be a source of free + // data availability, which the leaders would be incentivized to abuse. + if message.proposal_payload.is_some() { + return Err(Error::ReproposalWithPayload); + }; + + hash + } + // This is a new proposal, so we need to verify it (i.e. execute it). + None => { + // Check that the payload is present. + let Some(ref payload) = message.proposal_payload else { + return Err(Error::MissingPayload); + }; + + if payload.len() > self.config.max_payload_size { + return Err(Error::ProposalOversizedPayload { + payload_size: payload.len(), + }); + } + + // Defensively assume that PayloadManager cannot verify proposal until the previous block is stored. + // Note that it doesn't mean that the block is actually available, as old blocks might get pruned or + // we might just have started from a snapshot state. It just means that we have the state of the chain + // up to the previous block. + if let Some(prev) = implied_block_number.prev() { + self.config + .block_store + .wait_until_persisted(&ctx.with_deadline(self.view_timeout), prev) + .await + .map_err(|_| Error::MissingPreviousPayload { prev_number: prev })?; + } + + // Execute the payload. + if let Err(err) = self + .config + .payload_manager + .verify(ctx, implied_block_number, payload) + .await + { + return Err(match err { + ctx::Error::Internal(err) => Error::InvalidPayload(err), + err @ ctx::Error::Canceled(_) => Error::Internal(err), + }); + } + + // The proposal is valid. We cache it, waiting for it to be committed. + self.block_proposal_cache + .entry(implied_block_number) + .or_default() + .insert(payload.hash(), payload.clone()); + + payload.hash() + } + }; + + // ----------- All checks finished. Now we process the message. -------------- + + // Metrics. We observe the latency of receiving a proposal measured + // from the start of this view. + metrics::METRICS + .proposal_latency + .observe_latency(ctx.now() - self.view_start); + + // Create our commit vote. + let commit_vote = validator::ReplicaCommit { + view: message.view(), + proposal: BlockHeader { + number: implied_block_number, + payload: block_hash, + }, + }; + + // Update the state machine. + self.view_number = message.view().number; + self.phase = validator::Phase::Commit; + self.high_vote = Some(commit_vote.clone()); + match &message.justification { + validator::ProposalJustification::Commit(qc) => self + .process_commit_qc(ctx, qc) + .await + .wrap("process_commit_qc()")?, + validator::ProposalJustification::Timeout(qc) => { + if let Some(high_qc) = qc.high_qc() { + self.process_commit_qc(ctx, high_qc) + .await + .wrap("process_commit_qc()")?; + } + self.high_timeout_qc = max(Some(qc.clone()), self.high_timeout_qc.clone()); + } + }; + + // Backup our state. + self.backup_state(ctx).await.wrap("backup_state()")?; + + // Broadcast our commit message. + let output_message = ConsensusInputMessage { + message: self + .config + .secret_key + .sign_msg(validator::ConsensusMsg::ReplicaCommit(commit_vote)), + }; + self.outbound_pipe.send(output_message.into()); + + Ok(()) + } +} diff --git a/node/actors/bft/src/chonky_bft/proposer.rs b/node/actors/bft/src/chonky_bft/proposer.rs new file mode 100644 index 00000000..4a6dd843 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/proposer.rs @@ -0,0 +1,104 @@ +use crate::{io::OutputMessage, metrics, Config}; +use std::sync::Arc; +use zksync_concurrency::{ctx, error::Wrap as _, sync}; +use zksync_consensus_network::io::ConsensusInputMessage; +use zksync_consensus_roles::validator; + +use super::VIEW_TIMEOUT_DURATION; + +/// The proposer loop is responsible for proposing new blocks to the network. It watches for new +/// justifications from the replica and if it is the leader for the view, it proposes a new block. +pub(crate) async fn run_proposer( + ctx: &ctx::Ctx, + cfg: Arc, + outbound_pipe: ctx::channel::UnboundedSender, + mut justification_watch: sync::watch::Receiver>, +) -> ctx::Result<()> { + loop { + // Wait for a new justification to be available. + let Some(justification) = sync::changed(ctx, &mut justification_watch).await?.clone() + else { + continue; + }; + + // If we are not the leader for this view, skip it. + if cfg.genesis().view_leader(justification.view().number) != cfg.secret_key.public() { + continue; + } + + // Create a proposal for the given justification, within the timeout. + let proposal = match create_proposal( + &ctx.with_timeout(VIEW_TIMEOUT_DURATION), + cfg.clone(), + justification, + ) + .await + { + Ok(proposal) => proposal, + Err(ctx::Error::Canceled(_)) => { + tracing::error!("run_proposer(): timed out while creating a proposal"); + continue; + } + Err(ctx::Error::Internal(err)) => { + tracing::error!("run_proposer(): internal error: {err:#}"); + return Err(ctx::Error::Internal(err)); + } + }; + + // Broadcast our proposal to all replicas (ourselves included). + let msg = cfg + .secret_key + .sign_msg(validator::ConsensusMsg::LeaderProposal(proposal)); + + outbound_pipe.send(ConsensusInputMessage { message: msg }.into()); + } +} + +/// Creates a proposal for the given justification. +pub(crate) async fn create_proposal( + ctx: &ctx::Ctx, + cfg: Arc, + justification: validator::ProposalJustification, +) -> ctx::Result { + // Get the block number and check if this must be a reproposal. + let (block_number, opt_block_hash) = justification.get_implied_block(cfg.genesis()); + + let proposal_payload = match opt_block_hash { + // There was some proposal last view that a subquorum of replicas + // voted for and could have been finalized. We need to repropose it. + Some(_) => None, + // The previous proposal was finalized, so we can propose a new block. + None => { + // Defensively assume that PayloadManager cannot propose until the previous block is stored. + if let Some(prev) = block_number.prev() { + cfg.block_store.wait_until_persisted(ctx, prev).await?; + } + + let payload = cfg + .payload_manager + .propose(ctx, block_number) + .await + .wrap("payload_manager.propose()")?; + + if payload.0.len() > cfg.max_payload_size { + return Err(anyhow::format_err!( + "proposed payload too large: got {}B, max {}B", + payload.0.len(), + cfg.max_payload_size + ) + .into()); + } + + metrics::METRICS + .proposal_payload_size + .observe(payload.0.len()); + + Some(payload) + } + }; + + Ok(validator::LeaderProposal { + proposal_payload, + justification, + }) +} diff --git a/node/actors/bft/src/chonky_bft/testonly.rs b/node/actors/bft/src/chonky_bft/testonly.rs new file mode 100644 index 00000000..2367d1e7 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/testonly.rs @@ -0,0 +1,306 @@ +use crate::testonly::RandomPayload; +use crate::{ + chonky_bft::{self, commit, new_view, proposal, timeout, StateMachine}, + io::OutputMessage, + Config, PayloadManager, +}; +use assert_matches::assert_matches; +use std::sync::Arc; +use zksync_concurrency::sync::prunable_mpsc; +use zksync_concurrency::{ctx, sync}; +use zksync_consensus_network as network; +use zksync_consensus_network::io::ConsensusReq; +use zksync_consensus_roles::validator; +use zksync_consensus_storage::{ + testonly::{in_memory, TestMemoryStorage}, + BlockStoreRunner, +}; +use zksync_consensus_utils::enum_util::Variant; + +pub(crate) const MAX_PAYLOAD_SIZE: usize = 1000; + +/// `UTHarness` provides various utilities for unit tests. +/// It is designed to simplify the setup and execution of test cases by encapsulating +/// common testing functionality. +/// +/// It should be instantiated once for every test case. +#[cfg(test)] +pub(crate) struct UTHarness { + pub(crate) replica: StateMachine, + pub(crate) keys: Vec, + pub(crate) outbound_pipe: ctx::channel::UnboundedReceiver, + pub(crate) inbound_pipe: prunable_mpsc::Sender, + pub(crate) _proposer_pipe: sync::watch::Receiver>, +} + +impl UTHarness { + /// Creates a new `UTHarness` with the specified validator set size. + pub(crate) async fn new( + ctx: &ctx::Ctx, + num_validators: usize, + ) -> (UTHarness, BlockStoreRunner) { + Self::new_with_payload_manager( + ctx, + num_validators, + Box::new(RandomPayload(MAX_PAYLOAD_SIZE)), + ) + .await + } + + /// Creates a new `UTHarness` with minimally-significant validator set size. + pub(crate) async fn new_many(ctx: &ctx::Ctx) -> (UTHarness, BlockStoreRunner) { + let num_validators = 6; + let (util, runner) = UTHarness::new(ctx, num_validators).await; + assert!(util.genesis().validators.max_faulty_weight() > 0); + (util, runner) + } + + pub(crate) async fn new_with_payload_manager( + ctx: &ctx::Ctx, + num_validators: usize, + payload_manager: Box, + ) -> (UTHarness, BlockStoreRunner) { + let rng = &mut ctx.rng(); + let setup = validator::testonly::Setup::new(rng, num_validators); + let store = TestMemoryStorage::new(ctx, &setup).await; + let (send, recv) = ctx::channel::unbounded(); + let (proposer_sender, proposer_receiver) = sync::watch::channel(None); + + let cfg = Arc::new(Config { + secret_key: setup.validator_keys[0].clone(), + block_store: store.blocks.clone(), + replica_store: Box::new(in_memory::ReplicaStore::default()), + payload_manager, + max_payload_size: MAX_PAYLOAD_SIZE, + }); + let (replica, input_pipe) = + StateMachine::start(ctx, cfg.clone(), send.clone(), proposer_sender) + .await + .unwrap(); + let mut this = UTHarness { + replica, + keys: setup.validator_keys.clone(), + outbound_pipe: recv, + inbound_pipe: input_pipe, + _proposer_pipe: proposer_receiver, + }; + + let timeout = this.new_replica_timeout(ctx).await; + this.process_replica_timeout_all(ctx, timeout).await; + + (this, store.runner) + } + + pub(crate) fn owner_key(&self) -> &validator::SecretKey { + &self.replica.config.secret_key + } + + pub(crate) fn leader_key(&self) -> validator::SecretKey { + let leader = self.view_leader(self.replica.view_number); + self.keys + .iter() + .find(|key| key.public() == leader) + .unwrap() + .clone() + } + + pub(crate) fn view(&self) -> validator::View { + validator::View { + genesis: self.genesis().hash(), + number: self.replica.view_number, + } + } + + pub(crate) fn view_leader(&self, view: validator::ViewNumber) -> validator::PublicKey { + self.genesis().view_leader(view) + } + + pub(crate) fn genesis(&self) -> &validator::Genesis { + self.replica.config.genesis() + } + + pub(crate) async fn new_leader_proposal(&self, ctx: &ctx::Ctx) -> validator::LeaderProposal { + let justification = self.replica.get_justification(); + chonky_bft::proposer::create_proposal(ctx, self.replica.config.clone(), justification) + .await + .unwrap() + } + + pub(crate) async fn new_replica_commit(&mut self, ctx: &ctx::Ctx) -> validator::ReplicaCommit { + let proposal = self.new_leader_proposal(ctx).await; + self.process_leader_proposal(ctx, self.leader_key().sign_msg(proposal)) + .await + .unwrap() + .msg + } + + pub(crate) async fn new_replica_timeout( + &mut self, + ctx: &ctx::Ctx, + ) -> validator::ReplicaTimeout { + self.replica.start_timeout(ctx).await.unwrap(); + self.try_recv().unwrap().msg + } + + pub(crate) async fn new_replica_new_view(&self) -> validator::ReplicaNewView { + let justification = self.replica.get_justification(); + validator::ReplicaNewView { justification } + } + + pub(crate) async fn new_commit_qc( + &mut self, + ctx: &ctx::Ctx, + mutate_fn: impl FnOnce(&mut validator::ReplicaCommit), + ) -> validator::CommitQC { + let mut msg = self.new_replica_commit(ctx).await; + mutate_fn(&mut msg); + let mut qc = validator::CommitQC::new(msg.clone(), self.genesis()); + for key in &self.keys { + qc.add(&key.sign_msg(msg.clone()), self.genesis()).unwrap(); + } + qc + } + + // #[allow(dead_code)] + // pub(crate) fn new_timeout_qc( + // &mut self, + // mutate_fn: impl FnOnce(&mut validator::ReplicaTimeout), + // ) -> validator::TimeoutQC { + // let mut msg = self.new_replica_timeout(); + // mutate_fn(&mut msg); + // let mut qc = validator::TimeoutQC::new(msg.view.clone()); + // for key in &self.keys { + // qc.add(&key.sign_msg(msg.clone()), self.genesis()).unwrap(); + // } + // qc + // } + + pub(crate) async fn process_leader_proposal( + &mut self, + ctx: &ctx::Ctx, + msg: validator::Signed, + ) -> Result, proposal::Error> { + self.replica.on_proposal(ctx, msg).await?; + Ok(self.try_recv().unwrap()) + } + + pub(crate) async fn process_replica_commit( + &mut self, + ctx: &ctx::Ctx, + msg: validator::Signed, + ) -> Result>, commit::Error> { + self.replica.on_commit(ctx, msg).await?; + Ok(self.try_recv()) + } + + pub(crate) async fn process_replica_timeout( + &mut self, + ctx: &ctx::Ctx, + msg: validator::Signed, + ) -> Result>, timeout::Error> { + self.replica.on_timeout(ctx, msg).await?; + Ok(self.try_recv()) + } + + pub(crate) async fn process_replica_new_view( + &mut self, + ctx: &ctx::Ctx, + msg: validator::Signed, + ) -> Result>, new_view::Error> { + self.replica.on_new_view(ctx, msg).await?; + Ok(self.try_recv()) + } + + pub(crate) async fn process_replica_commit_all( + &mut self, + ctx: &ctx::Ctx, + msg: validator::ReplicaCommit, + ) -> validator::Signed { + let mut threshold_reached = false; + let mut cur_weight = 0; + + for key in self.keys.iter() { + let res = self.replica.on_commit(ctx, key.sign_msg(msg.clone())).await; + let val_index = self.genesis().validators.index(&key.public()).unwrap(); + + cur_weight += self.genesis().validators.get(val_index).unwrap().weight; + + if threshold_reached { + assert_matches!(res, Err(commit::Error::Old { .. })); + } else { + res.unwrap(); + if cur_weight >= self.genesis().validators.quorum_threshold() { + threshold_reached = true; + } + } + } + + self.try_recv().unwrap() + } + + pub(crate) async fn process_replica_timeout_all( + &mut self, + ctx: &ctx::Ctx, + msg: validator::ReplicaTimeout, + ) -> validator::Signed { + let mut threshold_reached = false; + let mut cur_weight = 0; + + for key in self.keys.iter() { + let res = self + .replica + .on_timeout(ctx, key.sign_msg(msg.clone())) + .await; + let val_index = self.genesis().validators.index(&key.public()).unwrap(); + + cur_weight += self.genesis().validators.get(val_index).unwrap().weight; + + if threshold_reached { + assert_matches!(res, Err(timeout::Error::Old { .. })); + } else { + res.unwrap(); + if cur_weight >= self.genesis().validators.quorum_threshold() { + threshold_reached = true; + } + } + } + + self.try_recv().unwrap() + } + + /// Produces a block, by executing the full view. + pub(crate) async fn produce_block(&mut self, ctx: &ctx::Ctx) { + let replica_commit = self.new_replica_commit(ctx).await; + self.process_replica_commit_all(ctx, replica_commit).await; + } + + /// Triggers replica timeout, processes the new validator::ReplicaTimeout + /// to start a new view, then executes the whole new view to make sure + /// that the consensus recovers after a timeout. + pub(crate) async fn produce_block_after_timeout(&mut self, ctx: &ctx::Ctx) { + let cur_view = self.replica.view_number; + + self.replica.start_timeout(ctx).await.unwrap(); + let replica_timeout = self.try_recv().unwrap().msg; + self.process_replica_timeout_all(ctx, replica_timeout).await; + + assert_eq!(self.replica.view_number, cur_view.next()); + + self.produce_block(ctx).await; + } + + pub(crate) fn send(&self, msg: validator::Signed) { + self.inbound_pipe.send(ConsensusReq { + msg, + ack: zksync_concurrency::oneshot::channel().0, + }); + } + + fn try_recv>(&mut self) -> Option> { + self.outbound_pipe.try_recv().map(|message| match message { + OutputMessage::Network(network::io::ConsensusInputMessage { message, .. }) => { + message.cast().unwrap() + } + }) + } +} diff --git a/node/actors/bft/src/chonky_bft/tests/commit.rs b/node/actors/bft/src/chonky_bft/tests/commit.rs new file mode 100644 index 00000000..0d65d110 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/tests/commit.rs @@ -0,0 +1,426 @@ +use crate::chonky_bft::{commit, testonly::UTHarness}; +use assert_matches::assert_matches; +use pretty_assertions::assert_eq; +use rand::Rng; +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::validator; + +#[tokio::test] +async fn commit_yield_new_view_sanity() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let cur_view = util.replica.view_number; + let replica_commit = util.new_replica_commit(ctx).await; + assert_eq!(util.replica.phase, validator::Phase::Commit); + + let new_view = util + .process_replica_commit_all(ctx, replica_commit.clone()) + .await + .msg; + assert_eq!(util.replica.view_number, cur_view.next()); + assert_eq!(util.replica.phase, validator::Phase::Prepare); + assert_eq!(new_view.view().number, cur_view.next()); + assert_matches!(new_view.justification, validator::ProposalJustification::Commit(qc) => { + assert_eq!(qc.message.proposal, replica_commit.proposal); + }); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn commit_non_validator_signer() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + let non_validator_key: validator::SecretKey = ctx.rng().gen(); + let res = util + .process_replica_commit(ctx, non_validator_key.sign_msg(replica_commit)) + .await; + + assert_matches!( + res, + Err(commit::Error::NonValidatorSigner { signer }) => { + assert_eq!(*signer, non_validator_key.public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_commit_old() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_commit = util.new_replica_commit(ctx).await; + replica_commit.view.number = validator::ViewNumber(util.replica.view_number.0 - 1); + let res = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit)) + .await; + + assert_matches!( + res, + Err(commit::Error::Old { current_view }) => { + assert_eq!(current_view, util.replica.view_number); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn commit_duplicate_signer() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_commit = util.new_replica_commit(ctx).await; + assert!(util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit.clone())) + .await + .unwrap() + .is_none()); + + // Processing twice same ReplicaCommit for same view gets DuplicateSigner error + let res = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit.clone())) + .await; + assert_matches!( + res, + Err(commit::Error::DuplicateSigner { + message_view, + signer + })=> { + assert_eq!(message_view, util.replica.view_number); + assert_eq!(*signer, util.owner_key().public()); + } + ); + + // Processing twice different ReplicaCommit for same view gets DuplicateSigner error too + replica_commit.proposal.number = replica_commit.proposal.number.next(); + let res = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit.clone())) + .await; + assert_matches!( + res, + Err(commit::Error::DuplicateSigner { + message_view, + signer + })=> { + assert_eq!(message_view, util.replica.view_number); + assert_eq!(*signer, util.owner_key().public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn commit_invalid_sig() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let msg = util.new_replica_commit(ctx).await; + let mut replica_commit = util.owner_key().sign_msg(msg); + replica_commit.sig = ctx.rng().gen(); + + let res = util.process_replica_commit(ctx, replica_commit).await; + assert_matches!(res, Err(commit::Error::InvalidSignature(..))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn commit_invalid_message() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_commit = util.new_replica_commit(ctx).await; + replica_commit.view.genesis = ctx.rng().gen(); + + let res = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit)) + .await; + assert_matches!(res, Err(commit::Error::InvalidMessage(_))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_commit_num_received_below_threshold() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + for i in 0..util.genesis().validators.quorum_threshold() as usize - 1 { + assert!(util + .process_replica_commit(ctx, util.keys[i].sign_msg(replica_commit.clone())) + .await + .unwrap() + .is_none()); + } + let res = util + .process_replica_commit( + ctx, + util.keys[util.genesis().validators.quorum_threshold() as usize - 1] + .sign_msg(replica_commit.clone()), + ) + .await + .unwrap() + .unwrap() + .msg; + assert_matches!(res.justification, validator::ProposalJustification::Commit(qc) => { + assert_eq!(qc.message.proposal, replica_commit.proposal); + }); + for i in util.genesis().validators.quorum_threshold() as usize..util.keys.len() { + let res = util + .process_replica_commit(ctx, util.keys[i].sign_msg(replica_commit.clone())) + .await; + assert_matches!(res, Err(commit::Error::Old { .. })); + } + + Ok(()) + }) + .await + .unwrap(); +} + +/// ReplicaCommit received before receiving LeaderProposal. +/// Whether replica accepts or rejects the message it doesn't matter. +/// It just shouldn't crash. +#[tokio::test] +async fn replica_commit_unexpected_proposal() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + util.produce_block(ctx).await; + let replica_commit = validator::ReplicaCommit { + view: util.view(), + proposal: validator::BlockHeader { + number: util + .replica + .high_commit_qc + .as_ref() + .unwrap() + .message + .proposal + .number + .next(), + payload: ctx.rng().gen(), + }, + }; + + let _ = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit)) + .await; + + Ok(()) + }) + .await + .unwrap(); +} + +/// Proposal should be the same for every ReplicaCommit +/// Check it doesn't fail if one validator sends a different proposal in +/// the ReplicaCommit +#[tokio::test] +async fn replica_commit_different_proposals() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + + // Process a modified replica_commit (ie. from a malicious or wrong node) + let mut bad_replica_commit = replica_commit.clone(); + bad_replica_commit.proposal.number = replica_commit.proposal.number.next(); + util.process_replica_commit(ctx, util.owner_key().sign_msg(bad_replica_commit)) + .await + .unwrap(); + + // The rest of the validators sign the correct one + let mut replica_commit_result = None; + for i in 1..util.keys.len() { + replica_commit_result = util + .process_replica_commit(ctx, util.keys[i].sign_msg(replica_commit.clone())) + .await + .unwrap(); + } + + // Check correct proposal has been committed + assert_matches!(replica_commit_result.unwrap().msg.justification, validator::ProposalJustification::Commit(qc) => { + assert_eq!(qc.message.proposal, replica_commit.proposal); + }); + + Ok(()) + }) + .await + .unwrap(); +} + +/// Check that leader won't accumulate undefined amount of messages if +/// it's spammed with ReplicaCommit messages for future views +#[tokio::test] +async fn replica_commit_limit_messages_in_memory() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_commit = util.new_replica_commit(ctx).await; + let mut view = util.view(); + // Spam it with 200 messages for different views + for _ in 0..200 { + replica_commit.view = view; + let res = util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit.clone())) + .await; + assert_matches!(res, Ok(_)); + view.number = view.number.next(); + } + + // Ensure only 1 commit_qc is in memory, as the previous 199 were discarded each time + // a new message was processed + assert_eq!(util.replica.commit_qcs_cache.len(), 1); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_commit_filter_functions_test() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + let msg = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit.clone(), + )); + + // Send a msg with invalid signature + let mut invalid_msg = msg.clone(); + invalid_msg.sig = ctx.rng().gen(); + util.send(invalid_msg); + + // Send a correct message + util.send(msg.clone()); + + // Validate only correct message is received + assert_eq!(util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, msg); + + // Send a msg with view number = 2 + let mut replica_commit_from_view_2 = replica_commit.clone(); + replica_commit_from_view_2.view.number = validator::ViewNumber(2); + let msg_from_view_2 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit_from_view_2, + )); + util.send(msg_from_view_2); + + // Send a msg with view number = 4, will prune message from view 2 + let mut replica_commit_from_view_4 = replica_commit.clone(); + replica_commit_from_view_4.view.number = validator::ViewNumber(4); + let msg_from_view_4 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit_from_view_4, + )); + util.send(msg_from_view_4.clone()); + + // Send a msg with view number = 3, will be discarded, as it is older than message from view 4 + let mut replica_commit_from_view_3 = replica_commit.clone(); + replica_commit_from_view_3.view.number = validator::ViewNumber(3); + let msg_from_view_3 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit_from_view_3, + )); + util.send(msg_from_view_3); + + // Validate only message from view 4 is received + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_view_4 + ); + + // Send a msg from validator 0 + let msg_from_validator_0 = util.keys[0].sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit.clone(), + )); + util.send(msg_from_validator_0.clone()); + + // Send a msg from validator 1 + let msg_from_validator_1 = util.keys[1].sign_msg(validator::ConsensusMsg::ReplicaCommit( + replica_commit.clone(), + )); + util.send(msg_from_validator_1.clone()); + + //Validate both are present in the inbound_pipe + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_validator_0 + ); + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_validator_1 + ); + + Ok(()) + }) + .await + .unwrap(); +} diff --git a/node/actors/bft/src/chonky_bft/tests/mod.rs b/node/actors/bft/src/chonky_bft/tests/mod.rs new file mode 100644 index 00000000..92a30580 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/tests/mod.rs @@ -0,0 +1,127 @@ +use crate::chonky_bft::testonly::UTHarness; +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::validator; + +mod commit; +mod new_view; +mod proposal; +//mod proposer; +mod timeout; + +/// Sanity check of the happy path. +#[tokio::test] +async fn block_production() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + util.produce_block(ctx).await; + + Ok(()) + }) + .await + .unwrap(); +} + +/// Sanity check of block production after timeout +#[tokio::test] +async fn block_production_timeout() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + util.produce_block_after_timeout(ctx).await; + + Ok(()) + }) + .await + .unwrap(); +} + +/// Sanity check of block production with reproposal. +#[tokio::test] +async fn block_production_timeout_reproposal() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + let mut timeout = util.new_replica_timeout(ctx).await; + + for i in 0..util.genesis().validators.subquorum_threshold() as usize { + util.process_replica_timeout(ctx, util.keys[i].sign_msg(timeout.clone())) + .await + .unwrap(); + } + timeout.high_vote = None; + for i in util.genesis().validators.subquorum_threshold() as usize..util.keys.len() { + let _ = util + .process_replica_timeout(ctx, util.keys[i].sign_msg(timeout.clone())) + .await; + } + + assert!(util.replica.high_commit_qc.is_none()); + util.produce_block(ctx).await; + assert_eq!( + util.replica.high_commit_qc.unwrap().message.proposal, + replica_commit.proposal + ); + + Ok(()) + }) + .await + .unwrap(); +} + +/// Testing liveness after the network becomes idle with replica in commit phase. +#[tokio::test] +async fn block_production_timeout_in_commit() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + util.new_replica_commit(ctx).await; + + // Replica is in `Phase::Commit`, but should still accept messages from newer views. + assert_eq!(util.replica.phase, validator::Phase::Commit); + util.produce_block_after_timeout(ctx).await; + + Ok(()) + }) + .await + .unwrap(); +} + +/// Testing liveness after the network becomes idle with replica having some cached commit messages for the current view. +#[tokio::test] +async fn block_production_timeout_some_commits() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let replica_commit = util.new_replica_commit(ctx).await; + assert!(util + .process_replica_commit(ctx, util.owner_key().sign_msg(replica_commit)) + .await + .unwrap() + .is_none()); + + // Replica is in `Phase::Commit`, but should still accept prepares from newer views. + assert_eq!(util.replica.phase, validator::Phase::Commit); + util.produce_block_after_timeout(ctx).await; + + Ok(()) + }) + .await + .unwrap(); +} diff --git a/node/actors/bft/src/chonky_bft/tests/new_view.rs b/node/actors/bft/src/chonky_bft/tests/new_view.rs new file mode 100644 index 00000000..7f1f6550 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/tests/new_view.rs @@ -0,0 +1,204 @@ +use crate::chonky_bft::{new_view, testonly::UTHarness}; +use assert_matches::assert_matches; +use rand::Rng; +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::validator; + +#[tokio::test] +async fn new_view_sanity() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let commit_1 = validator::ReplicaCommit { + view: util.view().next(), + proposal: validator::BlockHeader { + number: validator::BlockNumber(1), + payload: ctx.rng().gen(), + }, + }; + let mut commit_qc_1 = validator::CommitQC::new(commit_1.clone(), util.genesis()); + for key in &util.keys { + commit_qc_1 + .add(&key.sign_msg(commit_1.clone()), util.genesis()) + .unwrap(); + } + let new_view_1 = validator::ReplicaNewView { + justification: validator::ProposalJustification::Commit(commit_qc_1.clone()), + }; + + let commit_2 = validator::ReplicaCommit { + view: commit_1.view.next(), + proposal: validator::BlockHeader { + number: commit_1.proposal.number.next(), + payload: ctx.rng().gen(), + }, + }; + let mut commit_qc_2 = validator::CommitQC::new(commit_2.clone(), util.genesis()); + for key in &util.keys { + commit_qc_2 + .add(&key.sign_msg(commit_2.clone()), util.genesis()) + .unwrap(); + } + let new_view_2 = validator::ReplicaNewView { + justification: validator::ProposalJustification::Commit(commit_qc_2.clone()), + }; + + let timeout = validator::ReplicaTimeout { + view: commit_2.view.next(), + high_vote: None, + high_qc: Some(commit_qc_2.clone()), + }; + let mut timeout_qc = validator::TimeoutQC::new(timeout.view); + for key in &util.keys { + timeout_qc + .add(&key.sign_msg(timeout.clone()), util.genesis()) + .unwrap(); + } + let new_view_3 = validator::ReplicaNewView { + justification: validator::ProposalJustification::Timeout(timeout_qc.clone()), + }; + + // Check that first new view with commit QC updates the view and high commit QC. + let res = util + .process_replica_new_view(ctx, util.owner_key().sign_msg(new_view_1.clone())) + .await + .unwrap() + .unwrap() + .msg; + assert_eq!(util.view(), new_view_1.view()); + assert_matches!(res.justification, validator::ProposalJustification::Commit(qc) => { + assert_eq!(util.replica.high_commit_qc.clone().unwrap(), qc); + }); + + // Check that the third new view with timeout QC updates the view, high timeout QC and high commit QC. + let res = util + .process_replica_new_view(ctx, util.owner_key().sign_msg(new_view_3.clone())) + .await + .unwrap() + .unwrap() + .msg; + assert_eq!(util.view(), new_view_3.view()); + assert_matches!(res.justification, validator::ProposalJustification::Timeout(qc) => { + assert_eq!(util.replica.high_timeout_qc.clone().unwrap(), qc); + assert_eq!(util.replica.high_commit_qc.clone().unwrap(), qc.high_qc().unwrap().clone()); + }); + + // Check that the second new view with commit QC is ignored and doesn't affect the state. + let res = util + .process_replica_new_view(ctx, util.owner_key().sign_msg(new_view_2.clone())) + .await; + assert_eq!(util.view(), new_view_3.view()); + assert_eq!(util.replica.high_timeout_qc.clone().unwrap(), timeout_qc); + assert_eq!( + util.replica.high_commit_qc.clone().unwrap(), + timeout_qc.high_qc().unwrap().clone() + ); + assert_matches!( + res, + Err(new_view::Error::Old { current_view }) => { + assert_eq!(current_view, util.replica.view_number); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn new_view_non_validator_signer() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let replica_new_view = util.new_replica_new_view().await; + let non_validator_key: validator::SecretKey = ctx.rng().gen(); + let res = util + .process_replica_new_view(ctx, non_validator_key.sign_msg(replica_new_view)) + .await; + + assert_matches!( + res, + Err(new_view::Error::NonValidatorSigner { signer }) => { + assert_eq!(*signer, non_validator_key.public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_new_view_old() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let replica_new_view = util.new_replica_new_view().await; + util.produce_block(ctx).await; + let res = util + .process_replica_new_view(ctx, util.owner_key().sign_msg(replica_new_view)) + .await; + + assert_matches!( + res, + Err(new_view::Error::Old { current_view }) => { + assert_eq!(current_view, util.replica.view_number); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn new_view_invalid_sig() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let msg = util.new_replica_new_view().await; + let mut replica_new_view = util.owner_key().sign_msg(msg); + replica_new_view.sig = ctx.rng().gen(); + + let res = util.process_replica_new_view(ctx, replica_new_view).await; + assert_matches!(res, Err(new_view::Error::InvalidSignature(..))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn new_view_invalid_message() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let res = util + .process_replica_new_view(ctx, util.owner_key().sign_msg(ctx.rng().gen())) + .await; + assert_matches!(res, Err(new_view::Error::InvalidMessage(_))); + + Ok(()) + }) + .await + .unwrap(); +} diff --git a/node/actors/bft/src/chonky_bft/tests/proposal.rs b/node/actors/bft/src/chonky_bft/tests/proposal.rs new file mode 100644 index 00000000..2c5d10dc --- /dev/null +++ b/node/actors/bft/src/chonky_bft/tests/proposal.rs @@ -0,0 +1,364 @@ +use crate::{ + chonky_bft::{ + proposal, + testonly::{UTHarness, MAX_PAYLOAD_SIZE}, + }, + testonly::RejectPayload, +}; +use assert_matches::assert_matches; +use rand::Rng; +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::validator; + +#[tokio::test] +async fn proposal_yield_replica_commit_sanity() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let proposal = util.new_leader_proposal(ctx).await; + let replica_commit = util + .process_leader_proposal(ctx, util.owner_key().sign_msg(proposal.clone())) + .await + .unwrap(); + + assert_eq!( + replica_commit.msg, + validator::ReplicaCommit { + view: proposal.view(), + proposal: validator::BlockHeader { + number: proposal.justification.get_implied_block(util.genesis()).0, + payload: proposal.proposal_payload.unwrap().hash() + }, + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_old_view() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let proposal = util.new_leader_proposal(ctx).await; + + util.replica.phase = validator::Phase::Commit; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal.clone())) + .await; + + assert_matches!( + res, + Err(proposal::Error::Old { current_view, current_phase }) => { + assert_eq!(current_view, util.replica.view_number); + assert_eq!(current_phase, util.replica.phase); + } + ); + + util.replica.phase = validator::Phase::Timeout; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal.clone())) + .await; + + assert_matches!( + res, + Err(proposal::Error::Old { current_view, current_phase }) => { + assert_eq!(current_view, util.replica.view_number); + assert_eq!(current_phase, util.replica.phase); + } + ); + + util.replica.phase = validator::Phase::Prepare; + util.replica.view_number = util.replica.view_number.next(); + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!( + res, + Err(proposal::Error::Old { current_view, current_phase }) => { + assert_eq!(current_view, util.replica.view_number); + assert_eq!(current_phase, util.replica.phase); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_invalid_leader() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let proposal = util.new_leader_proposal(ctx).await; + + assert_ne!( + util.view_leader(proposal.view().number), + util.owner_key().public() + ); + + let res = util + .process_leader_proposal(ctx, util.owner_key().sign_msg(proposal)) + .await; + + assert_matches!( + res, + Err(proposal::Error::InvalidLeader { correct_leader, received_leader }) => { + assert_eq!(correct_leader, util.keys[1].public()); + assert_eq!(received_leader, util.keys[0].public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_invalid_signature() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let proposal = util.new_leader_proposal(ctx).await; + let mut signed_proposal = util.leader_key().sign_msg(proposal); + signed_proposal.sig = ctx.rng().gen(); + + let res = util.process_leader_proposal(ctx, signed_proposal).await; + + assert_matches!(res, Err(proposal::Error::InvalidSignature(_))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_invalid_message() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let mut proposal = util.new_leader_proposal(ctx).await; + proposal.justification = ctx.rng().gen(); + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!(res, Err(proposal::Error::InvalidMessage(_))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_pruned_block() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let fake_commit = validator::ReplicaCommit { + view: util.view(), + proposal: validator::BlockHeader { + number: util + .replica + .config + .block_store + .queued() + .first + .prev() + .unwrap() + .prev() + .unwrap(), + payload: ctx.rng().gen(), + }, + }; + + util.process_replica_commit_all(ctx, fake_commit).await; + + // The replica should now produce a proposal for an already pruned block number. + let proposal = util.new_leader_proposal(ctx).await; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!(res, Err(proposal::Error::ProposalAlreadyPruned)); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_reproposal_with_payload() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + util.new_replica_commit(ctx).await; + let replica_timeout = util.new_replica_timeout(ctx).await; + util.process_replica_timeout_all(ctx, replica_timeout).await; + + let mut proposal = util.new_leader_proposal(ctx).await; + assert!(proposal.proposal_payload.is_none()); + proposal.proposal_payload = Some(ctx.rng().gen()); + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!(res, Err(proposal::Error::ReproposalWithPayload)); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_missing_payload() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let mut proposal = util.new_leader_proposal(ctx).await; + proposal.proposal_payload = None; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!(res, Err(proposal::Error::MissingPayload)); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_proposal_oversized_payload() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let payload = validator::Payload(vec![0; MAX_PAYLOAD_SIZE + 1]); + let mut proposal = util.new_leader_proposal(ctx).await; + proposal.proposal_payload = Some(payload); + + let res = util + .process_leader_proposal(ctx, util.owner_key().sign_msg(proposal)) + .await; + assert_matches!( + res, + Err(proposal::Error::ProposalOversizedPayload{ payload_size }) => { + assert_eq!(payload_size, MAX_PAYLOAD_SIZE + 1); + } + ); + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_missing_previous_payload() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let missing_payload_number = util.replica.config.block_store.queued().first.next(); + let fake_commit = validator::ReplicaCommit { + view: util.view(), + proposal: validator::BlockHeader { + number: missing_payload_number, + payload: ctx.rng().gen(), + }, + }; + + util.process_replica_commit_all(ctx, fake_commit).await; + + let proposal = validator::LeaderProposal { + proposal_payload: Some(ctx.rng().gen()), + justification: validator::ProposalJustification::Commit( + util.replica.high_commit_qc.clone().unwrap(), + ), + }; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!( + res, + Err(proposal::Error::MissingPreviousPayload { prev_number } ) => { + assert_eq!(prev_number, missing_payload_number); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn proposal_invalid_payload() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = + UTHarness::new_with_payload_manager(ctx, 1, Box::new(RejectPayload)).await; + s.spawn_bg(runner.run(ctx)); + + let proposal = util.new_leader_proposal(ctx).await; + + let res = util + .process_leader_proposal(ctx, util.leader_key().sign_msg(proposal)) + .await; + + assert_matches!(res, Err(proposal::Error::InvalidPayload(_))); + + Ok(()) + }) + .await + .unwrap(); +} diff --git a/node/actors/bft/src/chonky_bft/tests/timeout.rs b/node/actors/bft/src/chonky_bft/tests/timeout.rs new file mode 100644 index 00000000..0ee2dc9f --- /dev/null +++ b/node/actors/bft/src/chonky_bft/tests/timeout.rs @@ -0,0 +1,439 @@ +use crate::chonky_bft::{testonly::UTHarness, timeout}; +use assert_matches::assert_matches; +use rand::Rng; +use zksync_concurrency::{ctx, scope}; +use zksync_consensus_roles::validator; + +#[tokio::test] +async fn timeout_yield_new_view_sanity() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let cur_view = util.replica.view_number; + let replica_timeout = util.new_replica_timeout(ctx).await; + assert_eq!(util.replica.phase, validator::Phase::Timeout); + + let new_view = util + .process_replica_timeout_all(ctx, replica_timeout.clone()) + .await + .msg; + assert_eq!(util.replica.view_number, cur_view.next()); + assert_eq!(util.replica.phase, validator::Phase::Prepare); + assert_eq!(new_view.view().number, cur_view.next()); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn timeout_non_validator_signer() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let replica_timeout = util.new_replica_timeout(ctx).await; + let non_validator_key: validator::SecretKey = ctx.rng().gen(); + let res = util + .process_replica_timeout(ctx, non_validator_key.sign_msg(replica_timeout)) + .await; + + assert_matches!( + res, + Err(timeout::Error::NonValidatorSigner { signer }) => { + assert_eq!(*signer, non_validator_key.public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_timeout_old() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_timeout = util.new_replica_timeout(ctx).await; + replica_timeout.view.number = validator::ViewNumber(util.replica.view_number.0 - 1); + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(replica_timeout)) + .await; + + assert_matches!( + res, + Err(timeout::Error::Old { current_view }) => { + assert_eq!(current_view, util.replica.view_number); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn timeout_duplicate_signer() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + util.produce_block(ctx).await; + + let replica_timeout = util.new_replica_timeout(ctx).await; + assert!(util + .process_replica_timeout(ctx, util.owner_key().sign_msg(replica_timeout.clone())) + .await + .unwrap() + .is_none()); + + // Processing twice same ReplicaTimeout for same view gets DuplicateSigner error + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(replica_timeout.clone())) + .await; + assert_matches!( + res, + Err(timeout::Error::DuplicateSigner { + message_view, + signer + })=> { + assert_eq!(message_view, util.replica.view_number); + assert_eq!(*signer, util.owner_key().public()); + } + ); + + // Processing twice different ReplicaTimeout for same view gets DuplicateSigner error too + // replica_timeout.high_vote = None; + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(replica_timeout.clone())) + .await; + assert_matches!( + res, + Err(timeout::Error::DuplicateSigner { + message_view, + signer + })=> { + assert_eq!(message_view, util.replica.view_number); + assert_eq!(*signer, util.owner_key().public()); + } + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn timeout_invalid_sig() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let msg = util.new_replica_timeout(ctx).await; + let mut replica_timeout = util.owner_key().sign_msg(msg); + replica_timeout.sig = ctx.rng().gen(); + + let res = util.process_replica_timeout(ctx, replica_timeout).await; + assert_matches!(res, Err(timeout::Error::InvalidSignature(..))); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn timeout_invalid_message() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 1).await; + s.spawn_bg(runner.run(ctx)); + + let replica_timeout = util.new_replica_timeout(ctx).await; + + let mut bad_replica_timeout = replica_timeout.clone(); + bad_replica_timeout.view.genesis = ctx.rng().gen(); + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(bad_replica_timeout)) + .await; + assert_matches!( + res, + Err(timeout::Error::InvalidMessage( + validator::ReplicaTimeoutVerifyError::BadView(_) + )) + ); + + let mut bad_replica_timeout = replica_timeout.clone(); + bad_replica_timeout.high_vote = Some(ctx.rng().gen()); + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(bad_replica_timeout)) + .await; + assert_matches!( + res, + Err(timeout::Error::InvalidMessage( + validator::ReplicaTimeoutVerifyError::InvalidHighVote(_) + )) + ); + + let mut bad_replica_timeout = replica_timeout.clone(); + bad_replica_timeout.high_qc = Some(ctx.rng().gen()); + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(bad_replica_timeout)) + .await; + assert_matches!( + res, + Err(timeout::Error::InvalidMessage( + validator::ReplicaTimeoutVerifyError::InvalidHighQC(_) + )) + ); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn timeout_num_received_below_threshold() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let replica_timeout = util.new_replica_timeout(ctx).await; + for i in 0..util.genesis().validators.quorum_threshold() as usize - 1 { + assert!(util + .process_replica_timeout(ctx, util.keys[i].sign_msg(replica_timeout.clone())) + .await + .unwrap() + .is_none()); + } + let res = util + .process_replica_timeout( + ctx, + util.keys[util.genesis().validators.quorum_threshold() as usize - 1] + .sign_msg(replica_timeout.clone()), + ) + .await + .unwrap() + .unwrap() + .msg; + assert_matches!(res.justification, validator::ProposalJustification::Timeout(qc) => { + assert_eq!(qc.view, replica_timeout.view); + }); + for i in util.genesis().validators.quorum_threshold() as usize..util.keys.len() { + let res = util + .process_replica_timeout(ctx, util.keys[i].sign_msg(replica_timeout.clone())) + .await; + assert_matches!(res, Err(timeout::Error::Old { .. })); + } + + Ok(()) + }) + .await + .unwrap(); +} + +/// Check all ReplicaTimeout are included for weight calculation +/// even on different messages for the same view. +#[tokio::test] +async fn timeout_weight_different_messages() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new_many(ctx).await; + s.spawn_bg(runner.run(ctx)); + + let view = util.view(); + util.produce_block(ctx).await; + + let replica_timeout = util.new_replica_timeout(ctx).await; + util.replica.phase = validator::Phase::Prepare; // To allow processing of proposal later. + let proposal = replica_timeout.clone().high_vote.unwrap().proposal; + + // Create a different proposal for the same view + let mut different_proposal = proposal; + different_proposal.number = different_proposal.number.next(); + + // Create a new ReplicaTimeout with the different proposal + let mut other_replica_timeout = replica_timeout.clone(); + let mut high_vote = other_replica_timeout.high_vote.clone().unwrap(); + high_vote.proposal = different_proposal; + let high_qc = util + .new_commit_qc(ctx, |msg: &mut validator::ReplicaCommit| { + msg.proposal = different_proposal; + msg.view = view; + }) + .await; + other_replica_timeout.high_vote = Some(high_vote); + other_replica_timeout.high_qc = Some(high_qc); + + let validators = util.keys.len(); + + // half of the validators sign replica_timeout + for i in 0..validators / 2 { + util.process_replica_timeout(ctx, util.keys[i].sign_msg(replica_timeout.clone())) + .await + .unwrap(); + } + + let mut res = None; + // The rest of the validators until threshold sign other_replica_timeout + for i in validators / 2..util.genesis().validators.quorum_threshold() as usize { + res = util + .process_replica_timeout(ctx, util.keys[i].sign_msg(other_replica_timeout.clone())) + .await + .unwrap(); + } + + assert_matches!(res.unwrap().msg.justification, validator::ProposalJustification::Timeout(qc) => { + assert_eq!(qc.view, replica_timeout.view); + assert_eq!(qc.high_vote(util.genesis()).unwrap(), proposal); + }); + + Ok(()) + }) + .await + .unwrap(); +} + +/// Check that leader won't accumulate undefined amount of messages if +/// it's spammed with ReplicaTimeout messages for future views +#[tokio::test] +async fn replica_timeout_limit_messages_in_memory() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let mut replica_timeout = util.new_replica_timeout(ctx).await; + let mut view = util.view(); + // Spam it with 200 messages for different views + for _ in 0..200 { + replica_timeout.view = view; + let res = util + .process_replica_timeout(ctx, util.owner_key().sign_msg(replica_timeout.clone())) + .await; + assert_matches!(res, Ok(_)); + view.number = view.number.next(); + } + + // Ensure only 1 timeout_qc is in memory, as the previous 199 were discarded each time + // a new message was processed + assert_eq!(util.replica.timeout_qcs_cache.len(), 1); + + Ok(()) + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn replica_timeout_filter_functions_test() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + scope::run!(ctx, |ctx, s| async { + let (mut util, runner) = UTHarness::new(ctx, 2).await; + s.spawn_bg(runner.run(ctx)); + + let replica_timeout = util.new_replica_timeout(ctx).await; + let msg = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout.clone(), + )); + + // Send a msg with invalid signature + let mut invalid_msg = msg.clone(); + invalid_msg.sig = ctx.rng().gen(); + util.send(invalid_msg); + + // Send a correct message + util.send(msg.clone()); + + // Validate only correct message is received + assert_eq!(util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, msg); + + // Send a msg with view number = 2 + let mut replica_timeout_from_view_2 = replica_timeout.clone(); + replica_timeout_from_view_2.view.number = validator::ViewNumber(2); + let msg_from_view_2 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout_from_view_2, + )); + util.send(msg_from_view_2); + + // Send a msg with view number = 4, will prune message from view 2 + let mut replica_timeout_from_view_4 = replica_timeout.clone(); + replica_timeout_from_view_4.view.number = validator::ViewNumber(4); + let msg_from_view_4 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout_from_view_4, + )); + util.send(msg_from_view_4.clone()); + + // Send a msg with view number = 3, will be discarded, as it is older than message from view 4 + let mut replica_timeout_from_view_3 = replica_timeout.clone(); + replica_timeout_from_view_3.view.number = validator::ViewNumber(3); + let msg_from_view_3 = util + .owner_key() + .sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout_from_view_3, + )); + util.send(msg_from_view_3); + + // Validate only message from view 4 is received + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_view_4 + ); + + // Send a msg from validator 0 + let msg_from_validator_0 = util.keys[0].sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout.clone(), + )); + util.send(msg_from_validator_0.clone()); + + // Send a msg from validator 1 + let msg_from_validator_1 = util.keys[1].sign_msg(validator::ConsensusMsg::ReplicaTimeout( + replica_timeout.clone(), + )); + util.send(msg_from_validator_1.clone()); + + // Validate both are present in the inbound_pipe + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_validator_0 + ); + assert_eq!( + util.replica.inbound_pipe.recv(ctx).await.unwrap().msg, + msg_from_validator_1 + ); + + Ok(()) + }) + .await + .unwrap(); +} diff --git a/node/actors/bft/src/chonky_bft/timeout.rs b/node/actors/bft/src/chonky_bft/timeout.rs new file mode 100644 index 00000000..189e3c16 --- /dev/null +++ b/node/actors/bft/src/chonky_bft/timeout.rs @@ -0,0 +1,187 @@ +use super::StateMachine; +use crate::metrics; +use std::{cmp::max, collections::HashSet}; +use zksync_concurrency::{ctx, error::Wrap, time}; +use zksync_consensus_network::io::ConsensusInputMessage; +use zksync_consensus_roles::validator; + +/// Errors that can occur when processing a ReplicaTimeout message. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + /// Message signer isn't part of the validator set. + #[error("message signer isn't part of the validator set (signer: {signer:?})")] + NonValidatorSigner { + /// Signer of the message. + signer: Box, + }, + /// Past view. + #[error("past view (current view: {current_view:?})")] + Old { + /// Current view. + current_view: validator::ViewNumber, + }, + /// Duplicate signer. We already have a timeout message from the same validator + /// for the same or past view. + #[error("duplicate signer (message view: {message_view:?}, signer: {signer:?})")] + DuplicateSigner { + /// View number of the message. + message_view: validator::ViewNumber, + /// Signer of the message. + signer: Box, + }, + /// Invalid message signature. + #[error("invalid signature: {0:#}")] + InvalidSignature(#[source] anyhow::Error), + /// Invalid message. + #[error("invalid message: {0:#}")] + InvalidMessage(#[source] validator::ReplicaTimeoutVerifyError), + /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. + #[error(transparent)] + Internal(#[from] ctx::Error), +} + +impl Wrap for Error { + fn with_wrap C>( + self, + f: F, + ) -> Self { + match self { + Error::Internal(err) => Error::Internal(err.with_wrap(f)), + err => err, + } + } +} + +impl StateMachine { + /// Processes a ReplicaTimeout message. + pub(crate) async fn on_timeout( + &mut self, + ctx: &ctx::Ctx, + signed_message: validator::Signed, + ) -> Result<(), Error> { + // ----------- Checking origin of the message -------------- + + // Unwrap message. + let message = &signed_message.msg; + let author = &signed_message.key; + + // Check that the message signer is in the validator committee. + if !self.config.genesis().validators.contains(author) { + return Err(Error::NonValidatorSigner { + signer: author.clone().into(), + }); + } + + // If the message is from a past view, ignore it. + if message.view.number < self.view_number { + return Err(Error::Old { + current_view: self.view_number, + }); + } + + // If we already have a message from the same validator for the same or past view, ignore it. + if let Some(&view) = self.timeout_views_cache.get(author) { + if view >= message.view.number { + return Err(Error::DuplicateSigner { + message_view: message.view.number, + signer: author.clone().into(), + }); + } + } + + // ----------- Checking the signed part of the message -------------- + + // Check the signature on the message. + signed_message.verify().map_err(Error::InvalidSignature)?; + + message + .verify(self.config.genesis()) + .map_err(Error::InvalidMessage)?; + + // ----------- All checks finished. Now we process the message. -------------- + + // We add the message to the incrementally-constructed QC. + let timeout_qc = self + .timeout_qcs_cache + .entry(message.view.number) + .or_insert_with(|| validator::TimeoutQC::new(message.view)); + + // Should always succeed as all checks have been already performed + timeout_qc + .add(&signed_message, self.config.genesis()) + .expect("could not add message to TimeoutQC"); + + // Calculate the TimeoutQC signers weight. + let weight = timeout_qc.weight(&self.config.genesis().validators); + + // Update view number of last timeout message for author + self.timeout_views_cache + .insert(author.clone(), message.view.number); + + // Clean up timeout_qcs for the case that no replica is at the view + // of a given TimeoutQC + // This prevents timeout_qcs map from growing indefinitely in case some + // malicious replica starts spamming messages for future views + let active_views: HashSet<_> = self.timeout_views_cache.values().collect(); + self.timeout_qcs_cache + .retain(|view_number, _| active_views.contains(view_number)); + + // Now we check if we have enough weight to continue. If not, we wait for more messages. + if weight < self.config.genesis().validators.quorum_threshold() { + return Ok(()); + }; + + // ----------- We have a QC. Now we process it. -------------- + + // Consume the created timeout QC for this view. + let timeout_qc = self.timeout_qcs_cache.remove(&message.view.number).unwrap(); + + // We update our state with the new timeout QC. + if let Some(commit_qc) = timeout_qc.high_qc() { + self.process_commit_qc(ctx, commit_qc) + .await + .wrap("process_commit_qc()")?; + } + self.high_timeout_qc = max(Some(timeout_qc.clone()), self.high_timeout_qc.clone()); + + // Start a new view. + self.start_new_view(ctx, message.view.number.next()).await?; + + Ok(()) + } + + /// This blocking method is used whenever we timeout in a view. + pub(crate) async fn start_timeout(&mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { + // Update the state machine. + self.phase = validator::Phase::Timeout; + self.view_timeout = time::Deadline::Infinite; + + // Backup our state. + self.backup_state(ctx).await.wrap("backup_state()")?; + + // Broadcast our timeout message. + let output_message = ConsensusInputMessage { + message: self + .config + .secret_key + .sign_msg(validator::ConsensusMsg::ReplicaTimeout( + validator::ReplicaTimeout { + view: validator::View { + genesis: self.config.genesis().hash(), + number: self.view_number, + }, + high_vote: self.high_vote.clone(), + high_qc: self.high_commit_qc.clone(), + }, + )), + }; + + self.outbound_pipe.send(output_message.into()); + + // Log the event. + tracing::info!("Timed out at view {}", self.view_number); + metrics::METRICS.replica_view_number.set(self.view_number.0); + + Ok(()) + } +} diff --git a/node/actors/bft/src/leader/mod.rs b/node/actors/bft/src/leader/mod.rs deleted file mode 100644 index f4615904..00000000 --- a/node/actors/bft/src/leader/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Implements the leader role in the Fastest-HotStuff consensus algorithm. The leader is the role that proposes blocks -//! and aggregates replica messages. It mainly acts as a central point of communication for the replicas. Note that -//! our consensus node will perform both the replica and leader roles simultaneously. - -pub(crate) mod replica_commit; -pub(crate) mod replica_prepare; -mod state_machine; -#[cfg(test)] -mod tests; - -pub(crate) use self::state_machine::StateMachine; diff --git a/node/actors/bft/src/leader/replica_commit.rs b/node/actors/bft/src/leader/replica_commit.rs deleted file mode 100644 index a28d8243..00000000 --- a/node/actors/bft/src/leader/replica_commit.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Handler of a ReplicaCommit message. - -use super::StateMachine; -use crate::metrics; -use std::collections::HashSet; -use zksync_concurrency::{ctx, metrics::LatencyHistogramExt as _}; -use zksync_consensus_network::io::{ConsensusInputMessage, Target}; -use zksync_consensus_roles::validator; - -/// Errors that can occur when processing a "replica commit" message. -#[derive(Debug, thiserror::Error)] -pub(crate) enum Error { - /// Message signer isn't part of the validator set. - #[error("Message signer isn't part of the validator set (signer: {signer:?})")] - NonValidatorSigner { - /// Signer of the message. - signer: Box, - }, - /// Past view or phase. - #[error("past view/phase (current view: {current_view:?}, current phase: {current_phase:?})")] - Old { - /// Current view. - current_view: validator::ViewNumber, - /// Current phase. - current_phase: validator::Phase, - }, - /// The processing node is not a lead for this message's view. - #[error("we are not a leader for this message's view")] - NotLeaderInView, - /// Invalid message. - #[error("invalid message: {0:#}")] - InvalidMessage(#[source] validator::ReplicaCommitVerifyError), - /// Invalid message signature. - #[error("invalid signature: {0:#}")] - InvalidSignature(#[source] anyhow::Error), -} - -impl StateMachine { - /// Processes `ReplicaCommit` message. - pub(crate) fn process_replica_commit( - &mut self, - ctx: &ctx::Ctx, - signed_message: validator::Signed, - ) -> Result<(), Error> { - // ----------- Checking origin of the message -------------- - - // Unwrap message. - let message = &signed_message.msg; - let author = &signed_message.key; - - // Check that the message signer is in the validator committee. - if !self.config.genesis().validators.contains(author) { - return Err(Error::NonValidatorSigner { - signer: author.clone().into(), - }); - } - - // If the message is from the "past", we discard it. - // That is, it's from a previous view or phase, or if we already received a message - // from the same validator and for the same view. - if (message.view.number, validator::Phase::Commit) < (self.view, self.phase) - || self - .replica_commit_views - .get(author) - .is_some_and(|view_number| *view_number >= message.view.number) - { - return Err(Error::Old { - current_view: self.view, - current_phase: self.phase, - }); - } - - // If the message is for a view when we are not a leader, we discard it. - if self.config.genesis().view_leader(message.view.number) != self.config.secret_key.public() - { - return Err(Error::NotLeaderInView); - } - - // ----------- Checking the signed part of the message -------------- - - // Check the signature on the message. - signed_message.verify().map_err(Error::InvalidSignature)?; - - message - .verify(self.config.genesis()) - .map_err(Error::InvalidMessage)?; - - // ----------- All checks finished. Now we process the message. -------------- - - // We add the message to the incrementally-constructed QC. - let commit_qc = self - .commit_qcs - .entry(message.view.number) - .or_default() - .entry(message.clone()) - .or_insert_with(|| validator::CommitQC::new(message.clone(), self.config.genesis())); - - // Should always succeed as all checks have been already performed - commit_qc - .add(&signed_message, self.config.genesis()) - .expect("Could not add message to CommitQC"); - - // Calculate the CommitQC signers weight. - let weight = self.config.genesis().validators.weight(&commit_qc.signers); - - // Update commit message current view number for author - self.replica_commit_views - .insert(author.clone(), message.view.number); - - // Clean up commit_qcs for the case that no replica is at the view - // of a given CommitQC - // This prevents commit_qcs map from growing indefinitely in case some - // malicious replica starts spamming messages for future views - let active_views: HashSet<_> = self.replica_commit_views.values().collect(); - self.commit_qcs - .retain(|view_number, _| active_views.contains(view_number)); - - // Now we check if we have enough weight to continue. - if weight < self.config.genesis().validators.threshold() { - return Ok(()); - }; - - // ----------- Update the state machine -------------- - let now = ctx.now(); - metrics::METRICS - .leader_commit_phase_latency - .observe_latency(now - self.phase_start); - self.view = message.view.number.next(); - self.phase = validator::Phase::Prepare; - self.phase_start = now; - - // ----------- Prepare our message and send it. -------------- - - // Consume the incrementally-constructed QC for this view. - let justification = self - .commit_qcs - .remove(&message.view.number) - .unwrap() - .remove(message) - .unwrap(); - - // Broadcast the leader commit message to all replicas (ourselves included). - let output_message = ConsensusInputMessage { - message: self - .config - .secret_key - .sign_msg(validator::ConsensusMsg::LeaderCommit( - validator::LeaderCommit { justification }, - )), - recipient: Target::Broadcast, - }; - self.outbound_pipe.send(output_message.into()); - - Ok(()) - } -} diff --git a/node/actors/bft/src/leader/replica_prepare.rs b/node/actors/bft/src/leader/replica_prepare.rs deleted file mode 100644 index 060caf77..00000000 --- a/node/actors/bft/src/leader/replica_prepare.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Handler of a ReplicaPrepare message. -use super::StateMachine; -use std::collections::HashSet; -use zksync_concurrency::{ctx, error::Wrap}; -use zksync_consensus_roles::validator; - -/// Errors that can occur when processing a "replica prepare" message. -#[derive(Debug, thiserror::Error)] -pub(crate) enum Error { - /// Message signer isn't part of the validator set. - #[error("Message signer isn't part of the validator set (signer: {signer:?})")] - NonValidatorSigner { - /// Signer of the message. - signer: validator::PublicKey, - }, - /// Past view or phase. - #[error("past view/phase (current view: {current_view:?}, current phase: {current_phase:?})")] - Old { - /// Current view. - current_view: validator::ViewNumber, - /// Current phase. - current_phase: validator::Phase, - }, - /// The node is not a leader for this message's view. - #[error("we are not a leader for this message's view")] - NotLeaderInView, - /// Invalid message signature. - #[error("invalid signature: {0:#}")] - InvalidSignature(#[source] anyhow::Error), - /// Invalid message. - #[error(transparent)] - InvalidMessage(validator::ReplicaPrepareVerifyError), - /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. - #[error(transparent)] - Internal(#[from] ctx::Error), -} - -impl Wrap for Error { - fn with_wrap C>( - self, - f: F, - ) -> Self { - match self { - Error::Internal(err) => Error::Internal(err.with_wrap(f)), - err => err, - } - } -} - -impl StateMachine { - /// Processes `ReplicaPrepare` message. - pub(crate) async fn process_replica_prepare( - &mut self, - ctx: &ctx::Ctx, - signed_message: validator::Signed, - ) -> Result<(), Error> { - // ----------- Checking origin of the message -------------- - - // Unwrap message. - let message = signed_message.msg.clone(); - let author = &signed_message.key; - - // Check that the message signer is in the validator set. - if !self.config.genesis().validators.contains(author) { - return Err(Error::NonValidatorSigner { - signer: author.clone(), - }); - } - - // If the message is from the "past", we discard it. - // That is, it's from a previous view or phase, or if we already received a message - // from the same validator and for the same view. - if (message.view.number, validator::Phase::Prepare) < (self.view, self.phase) - || self - .replica_prepare_views - .get(author) - .is_some_and(|view_number| *view_number >= message.view.number) - { - return Err(Error::Old { - current_view: self.view, - current_phase: self.phase, - }); - } - - // If the message is for a view when we are not a leader, we discard it. - if self.config.genesis().view_leader(message.view.number) != self.config.secret_key.public() - { - return Err(Error::NotLeaderInView); - } - - // ----------- Checking the signed part of the message -------------- - - // Check the signature on the message. - signed_message.verify().map_err(Error::InvalidSignature)?; - - // Verify the message. - message - .verify(self.config.genesis()) - .map_err(Error::InvalidMessage)?; - - // ----------- All checks finished. Now we process the message. -------------- - - // We add the message to the incrementally-constructed QC. - let prepare_qc = self - .prepare_qcs - .entry(message.view.number) - .or_insert_with(|| validator::PrepareQC::new(message.view.clone())); - - // Should always succeed as all checks have been already performed - prepare_qc - .add(&signed_message, self.config.genesis()) - .expect("Could not add message to PrepareQC"); - - // Calculate the PrepareQC signers weight. - let weight = prepare_qc.weight(&self.config.genesis().validators); - - // Update prepare message current view number for author - self.replica_prepare_views - .insert(author.clone(), message.view.number); - - // Clean up prepare_qcs for the case that no replica is at the view - // of a given PrepareQC - // This prevents prepare_qcs map from growing indefinitely in case some - // malicious replica starts spamming messages for future views - let active_views: HashSet<_> = self.replica_prepare_views.values().collect(); - self.prepare_qcs - .retain(|view_number, _| active_views.contains(view_number)); - - // Now we check if we have enough weight to continue. - if weight < self.config.genesis().validators.threshold() { - return Ok(()); - } - - // ----------- Update the state machine -------------- - - self.view = message.view.number; - self.phase = validator::Phase::Commit; - self.phase_start = ctx.now(); - - // Consume the incrementally-constructed QC for this view. - let justification = self.prepare_qcs.remove(&message.view.number).unwrap(); - - self.prepare_qc.send_replace(Some(justification)); - Ok(()) - } -} diff --git a/node/actors/bft/src/leader/state_machine.rs b/node/actors/bft/src/leader/state_machine.rs deleted file mode 100644 index 9e668751..00000000 --- a/node/actors/bft/src/leader/state_machine.rs +++ /dev/null @@ -1,270 +0,0 @@ -use crate::{metrics, Config, OutputSender}; -use std::{collections::BTreeMap, sync::Arc, unreachable}; -use zksync_concurrency::{ - ctx, - error::Wrap as _, - metrics::LatencyHistogramExt as _, - sync::{self, prunable_mpsc::SelectionFunctionResult}, - time, -}; -use zksync_consensus_network::io::{ConsensusInputMessage, ConsensusReq, Target}; -use zksync_consensus_roles::validator; - -/// The StateMachine struct contains the state of the leader. This is a simple state machine. We just store -/// replica messages and produce leader messages (including proposing blocks) when we reach the threshold for -/// those messages. When participating in consensus we are not the leader most of the time. -pub(crate) struct StateMachine { - /// Consensus configuration and output channel. - pub(crate) config: Arc, - /// Pipe through which leader sends network messages. - pub(crate) outbound_pipe: OutputSender, - /// Pipe through which leader receives network requests. - pub(crate) inbound_pipe: sync::prunable_mpsc::Receiver, - /// The current view number. This might not match the replica's view number, we only have this here - /// to make the leader advance monotonically in time and stop it from accepting messages from the past. - pub(crate) view: validator::ViewNumber, - /// The current phase. This might not match the replica's phase, we only have this here - /// to make the leader advance monotonically in time and stop it from accepting messages from the past. - pub(crate) phase: validator::Phase, - /// Time when the current phase has started. - pub(crate) phase_start: time::Instant, - /// Latest view each validator has signed a ReplicaPrepare message for. - pub(crate) replica_prepare_views: BTreeMap, - /// Prepare QCs indexed by view number. - pub(crate) prepare_qcs: BTreeMap, - /// Newest prepare QC composed from the `ReplicaPrepare` messages. - pub(crate) prepare_qc: sync::watch::Sender>, - /// Commit QCs indexed by view number and then by message. - pub(crate) commit_qcs: - BTreeMap>, - /// Latest view each validator has signed a ReplicaCommit message for. - pub(crate) replica_commit_views: BTreeMap, -} - -impl StateMachine { - /// Creates a new [`StateMachine`] instance. - /// - /// Returns a tuple containing: - /// * The newly created [`StateMachine`] instance. - /// * A sender handle that should be used to send values to be processed by the instance, asynchronously. - pub(crate) fn new( - ctx: &ctx::Ctx, - config: Arc, - outbound_pipe: OutputSender, - ) -> (Self, sync::prunable_mpsc::Sender) { - let (send, recv) = sync::prunable_mpsc::channel( - StateMachine::inbound_filter_predicate, - StateMachine::inbound_selection_function, - ); - - let this = StateMachine { - config, - outbound_pipe, - view: validator::ViewNumber(0), - phase: validator::Phase::Prepare, - phase_start: ctx.now(), - replica_prepare_views: BTreeMap::new(), - prepare_qcs: BTreeMap::new(), - prepare_qc: sync::watch::channel(None).0, - commit_qcs: BTreeMap::new(), - inbound_pipe: recv, - replica_commit_views: BTreeMap::new(), - }; - - (this, send) - } - - /// Runs a loop to process incoming messages. - /// This is the main entry point for the state machine, - /// potentially triggering state modifications and message sending to the executor. - pub(crate) async fn run(mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { - loop { - let req = self.inbound_pipe.recv(ctx).await?; - - let now = ctx.now(); - use validator::ConsensusMsg as M; - let label = match &req.msg.msg { - M::ReplicaPrepare(_) => { - let res = match self - .process_replica_prepare(ctx, req.msg.cast().unwrap()) - .await - .wrap("process_replica_prepare()") - { - Ok(()) => Ok(()), - Err(err) => { - match err { - super::replica_prepare::Error::Internal(e) => { - tracing::error!( - "process_replica_prepare: internal error: {e:#}" - ); - - return Err(e); - } - super::replica_prepare::Error::Old { .. } - | super::replica_prepare::Error::NotLeaderInView => { - // It's broadcasted now, so everyone gets it. - tracing::debug!("process_replica_prepare: {err:#}"); - } - _ => { - tracing::warn!("process_replica_prepare: {err:#}"); - } - } - Err(()) - } - }; - metrics::ConsensusMsgLabel::ReplicaPrepare.with_result(&res) - } - M::ReplicaCommit(_) => { - let res = self - .process_replica_commit(ctx, req.msg.cast().unwrap()) - .map_err(|err| { - tracing::warn!("process_replica_commit: {err:#}"); - }); - metrics::ConsensusMsgLabel::ReplicaCommit.with_result(&res) - } - _ => unreachable!(), - }; - metrics::METRICS.leader_processing_latency[&label].observe_latency(ctx.now() - now); - - // Notify network actor that the message has been processed. - // Ignore sending error. - let _ = req.ack.send(()); - } - } - - /// In a loop, receives a PrepareQC and sends a LeaderPrepare containing it. - /// Every subsequent PrepareQC has to be for a higher view than the previous one (otherwise it - /// is skipped). In case payload generation takes too long, some PrepareQC may be elided, so - /// that the validator doesn't spend time on generating payloads for already expired views. - pub(crate) async fn run_proposer( - ctx: &ctx::Ctx, - config: &Config, - mut prepare_qc: sync::watch::Receiver>, - pipe: &OutputSender, - ) -> ctx::Result<()> { - let mut next_view = validator::ViewNumber(0); - loop { - let Some(prepare_qc) = sync::changed(ctx, &mut prepare_qc).await?.clone() else { - continue; - }; - if prepare_qc.view.number < next_view { - continue; - }; - next_view = prepare_qc.view.number.next(); - Self::propose(ctx, config, prepare_qc, pipe) - .await - .wrap("propose()")?; - } - } - - /// Sends a LeaderPrepare for the given PrepareQC. - /// Uses `payload_source` to generate a payload if needed. - pub(crate) async fn propose( - ctx: &ctx::Ctx, - cfg: &Config, - justification: validator::PrepareQC, - pipe: &OutputSender, - ) -> ctx::Result<()> { - let high_vote = justification.high_vote(cfg.genesis()); - let high_qc = justification.high_qc(); - - // Create the block proposal to send to the replicas, - // and the commit vote to store in our block proposal cache. - let (proposal, payload) = match high_vote { - // The previous block was not finalized, so we need to propose it again. - // For this we only need the header, since we are guaranteed that at least - // f+1 honest replicas have the block and can broadcast it when finalized - // (2f+1 have stated that they voted for the block, at most f are malicious). - Some(proposal) if Some(&proposal) != high_qc.map(|qc| &qc.message.proposal) => { - (proposal, None) - } - // The previous block was finalized, so we can propose a new block. - _ => { - let number = match high_qc { - Some(qc) => qc.header().number.next(), - None => cfg.genesis().first_block, - }; - // Defensively assume that PayloadManager cannot propose until the previous block is stored. - if let Some(prev) = number.prev() { - cfg.block_store.wait_until_persisted(ctx, prev).await?; - } - let payload = cfg - .payload_manager - .propose(ctx, number) - .await - .wrap("payload_manager.propose()")?; - if payload.0.len() > cfg.max_payload_size { - return Err(anyhow::format_err!( - "proposed payload too large: got {}B, max {}B", - payload.0.len(), - cfg.max_payload_size - ) - .into()); - } - metrics::METRICS - .leader_proposal_payload_size - .observe(payload.0.len()); - let proposal = validator::BlockHeader { - number, - payload: payload.hash(), - }; - (proposal, Some(payload)) - } - }; - - // ----------- Prepare our message and send it -------------- - - // Broadcast the leader prepare message to all replicas (ourselves included). - let msg = cfg - .secret_key - .sign_msg(validator::ConsensusMsg::LeaderPrepare( - validator::LeaderPrepare { - proposal, - proposal_payload: payload, - justification, - }, - )); - pipe.send( - ConsensusInputMessage { - message: msg, - recipient: Target::Broadcast, - } - .into(), - ); - Ok(()) - } - - fn inbound_filter_predicate(new_req: &ConsensusReq) -> bool { - // Verify message signature - new_req.msg.verify().is_ok() - } - - fn inbound_selection_function( - old_req: &ConsensusReq, - new_req: &ConsensusReq, - ) -> SelectionFunctionResult { - if old_req.msg.key != new_req.msg.key { - return SelectionFunctionResult::Keep; - } - use validator::ConsensusMsg as M; - match (&old_req.msg.msg, &new_req.msg.msg) { - (M::ReplicaPrepare(old), M::ReplicaPrepare(new)) => { - // Discard older message - if old.view.number < new.view.number { - SelectionFunctionResult::DiscardOld - } else { - SelectionFunctionResult::DiscardNew - } - } - (M::ReplicaCommit(old), M::ReplicaCommit(new)) => { - // Discard older message - if old.view.number < new.view.number { - SelectionFunctionResult::DiscardOld - } else { - SelectionFunctionResult::DiscardNew - } - } - _ => SelectionFunctionResult::Keep, - } - } -} diff --git a/node/actors/bft/src/leader/tests.rs b/node/actors/bft/src/leader/tests.rs deleted file mode 100644 index 6b03681d..00000000 --- a/node/actors/bft/src/leader/tests.rs +++ /dev/null @@ -1,914 +0,0 @@ -use super::*; -use crate::testonly::ut_harness::UTHarness; -use assert_matches::assert_matches; -use pretty_assertions::assert_eq; -use rand::Rng; -use zksync_concurrency::{ctx, scope}; -use zksync_consensus_roles::validator::{self, Phase, ViewNumber}; - -#[tokio::test] -async fn replica_prepare_sanity() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - tracing::info!("started"); - util.new_leader_prepare(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_sanity_yield_leader_prepare() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let replica_prepare = util.new_replica_prepare(); - let leader_prepare = util - .process_replica_prepare(ctx, util.sign(replica_prepare.clone())) - .await - .unwrap() - .unwrap(); - assert_eq!(leader_prepare.msg.view(), &replica_prepare.view); - assert_eq!( - leader_prepare.msg.justification, - util.new_prepare_qc(|msg| *msg = replica_prepare) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_sanity_yield_leader_prepare_reproposal() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.new_replica_commit(ctx).await; - util.process_replica_timeout(ctx).await; - let replica_prepare = util.new_replica_prepare(); - let leader_prepare = util - .process_replica_prepare_all(ctx, replica_prepare.clone()) - .await; - - assert_eq!(leader_prepare.msg.view(), &replica_prepare.view); - assert_eq!( - Some(leader_prepare.msg.proposal), - replica_prepare.high_vote.as_ref().map(|v| v.proposal), - ); - assert_eq!(leader_prepare.msg.proposal_payload, None); - let map = leader_prepare.msg.justification.map; - assert_eq!(map.len(), 1); - assert_eq!(*map.first_key_value().unwrap().0, replica_prepare); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_bad_chain() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_prepare = util.new_replica_prepare(); - replica_prepare.view.genesis = rng.gen(); - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::InvalidMessage( - validator::ReplicaPrepareVerifyError::View(_) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_non_validator_signer() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - let non_validator_key: validator::SecretKey = ctx.rng().gen(); - let res = util - .process_replica_prepare(ctx, non_validator_key.sign_msg(replica_prepare)) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::NonValidatorSigner { signer }) => { - assert_eq!(signer, non_validator_key.public()); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_old_view() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - util.leader.view = util.replica.view.next(); - util.leader.phase = Phase::Prepare; - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::Old { - current_view: ViewNumber(2), - current_phase: Phase::Prepare, - }) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_during_commit() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - util.leader.view = util.replica.view; - util.leader.phase = Phase::Commit; - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::Old { - current_view, - current_phase: Phase::Commit, - }) => { - assert_eq!(current_view, util.replica.view); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_not_leader_in_view() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_prepare = util.new_replica_prepare(); - replica_prepare.view.number = replica_prepare.view.number.next(); - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await; - assert_matches!(res, Err(replica_prepare::Error::NotLeaderInView)); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_already_exists() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - util.set_owner_as_view_leader(); - let replica_prepare = util.new_replica_prepare(); - let replica_prepare = util.sign(replica_prepare.clone()); - assert!(util - .process_replica_prepare(ctx, replica_prepare.clone()) - .await - .unwrap() - .is_none()); - let res = util - .process_replica_prepare(ctx, replica_prepare.clone()) - .await; - assert_matches!(res, Err(replica_prepare::Error::Old { .. })); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_num_received_below_threshold() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - util.set_owner_as_view_leader(); - let replica_prepare = util.new_replica_prepare(); - assert!(util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await - .unwrap() - .is_none()); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_invalid_sig() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let msg = util.new_replica_prepare(); - let mut replica_prepare = util.sign(msg); - replica_prepare.sig = ctx.rng().gen(); - let res = util.process_replica_prepare(ctx, replica_prepare).await; - assert_matches!(res, Err(replica_prepare::Error::InvalidSignature(_))); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_invalid_commit_qc() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let mut replica_prepare = util.new_replica_prepare(); - replica_prepare.high_qc.as_mut().unwrap().signature = rng.gen(); - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::InvalidMessage( - validator::ReplicaPrepareVerifyError::HighQC(_) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -/// Check that leader behaves correctly in case receiving ReplicaPrepare -/// with high_qc with future views (which shouldn't be available yet). -#[tokio::test] -async fn replica_prepare_high_qc_of_future_view() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let mut view = util.replica_view(); - let mut replica_prepare = util.new_replica_prepare(); - // Check both the current view and next view. - for _ in 0..2 { - let qc = util.new_commit_qc(|msg| msg.view = view.clone()); - replica_prepare.high_qc = Some(qc); - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare.clone())) - .await; - assert_matches!( - res, - Err(replica_prepare::Error::InvalidMessage( - validator::ReplicaPrepareVerifyError::HighQCFutureView - )) - ); - view.number = view.number.next(); - } - Ok(()) - }) - .await - .unwrap(); -} - -/// Check all ReplicaPrepare are included for weight calculation -/// even on different messages for the same view. -#[tokio::test] -async fn replica_prepare_different_messages() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - - let view = util.replica_view(); - let replica_prepare = util.new_replica_prepare(); - - // Create a different proposal for the same view - let proposal = replica_prepare.clone().high_vote.unwrap().proposal; - let mut different_proposal = proposal; - different_proposal.number = different_proposal.number.next(); - - // Create a new ReplicaPrepare with the different proposal - let mut other_replica_prepare = replica_prepare.clone(); - let mut high_vote = other_replica_prepare.high_vote.clone().unwrap(); - high_vote.proposal = different_proposal; - let high_qc = util.new_commit_qc(|msg| { - msg.proposal = different_proposal; - msg.view = view.clone() - }); - - other_replica_prepare.high_vote = Some(high_vote); - other_replica_prepare.high_qc = Some(high_qc); - - let validators = util.keys.len(); - - // half of the validators sign replica_prepare - for i in 0..validators / 2 { - util.process_replica_prepare(ctx, util.keys[i].sign_msg(replica_prepare.clone())) - .await - .unwrap(); - } - - let mut replica_commit_result = None; - // The rest of the validators until threshold sign other_replica_prepare - for i in validators / 2..util.genesis().validators.threshold() as usize { - replica_commit_result = util - .process_replica_prepare(ctx, util.keys[i].sign_msg(other_replica_prepare.clone())) - .await - .unwrap(); - } - - // That should be enough for a proposal to be committed (even with different proposals) - assert_matches!(replica_commit_result, Some(_)); - - // Check the first proposal has been committed (as it has more votes) - let message = replica_commit_result.unwrap().msg; - assert_eq!(message.proposal, proposal); - Ok(()) - }) - .await - .unwrap(); -} - -/// Check that leader won't accumulate undefined amount of messages if -/// it's spammed with ReplicaPrepare messages for future views -#[tokio::test] -async fn replica_prepare_limit_messages_in_memory() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_prepare = util.new_replica_prepare(); - let mut view = util.replica_view(); - // Spam it with 200 messages for different views - for _ in 0..200 { - replica_prepare.view = view.clone(); - let res = util - .process_replica_prepare(ctx, util.sign(replica_prepare.clone())) - .await; - assert_matches!(res, Ok(_)); - // Since we have 2 replicas, we have to send only even numbered views - // to hit the same leader (the other replica will be leader on odd numbered views) - view.number = view.number.next().next(); - } - // Ensure only 1 prepare_qc is in memory, as the previous 199 were discarded each time - // new message is processed - assert_eq!(util.leader.prepare_qcs.len(), 1); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_prepare_filter_functions_test() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - let msg = util.sign(validator::ConsensusMsg::ReplicaPrepare( - replica_prepare.clone(), - )); - - // Send a msg with invalid signature - let mut invalid_msg = msg.clone(); - invalid_msg.sig = ctx.rng().gen(); - util.leader_send(invalid_msg); - - // Send a correct message - util.leader_send(msg.clone()); - - // Validate only correct message is received - assert_eq!(util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, msg); - - // Send a msg with view number = 2 - let mut replica_commit_from_view_2 = replica_prepare.clone(); - replica_commit_from_view_2.view.number = ViewNumber(2); - let msg_from_view_2 = util.sign(validator::ConsensusMsg::ReplicaPrepare( - replica_commit_from_view_2, - )); - util.leader_send(msg_from_view_2); - - // Send a msg with view number = 4, will prune message from view 2 - let mut replica_commit_from_view_4 = replica_prepare.clone(); - replica_commit_from_view_4.view.number = ViewNumber(4); - let msg_from_view_4 = util.sign(validator::ConsensusMsg::ReplicaPrepare( - replica_commit_from_view_4, - )); - util.leader_send(msg_from_view_4.clone()); - - // Send a msg with view number = 3, will be discarded, as it is older than message from view 4 - let mut replica_commit_from_view_3 = replica_prepare.clone(); - replica_commit_from_view_3.view.number = ViewNumber(3); - let msg_from_view_3 = util.sign(validator::ConsensusMsg::ReplicaPrepare( - replica_commit_from_view_3, - )); - util.leader_send(msg_from_view_3); - - // Validate only message from view 4 is received - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_view_4 - ); - - // Send a msg from validator 0 - let msg_from_validator_0 = util.keys[0].sign_msg(validator::ConsensusMsg::ReplicaPrepare( - replica_prepare.clone(), - )); - util.leader_send(msg_from_validator_0.clone()); - - // Send a msg from validator 1 - let msg_from_validator_1 = util.keys[1].sign_msg(validator::ConsensusMsg::ReplicaPrepare( - replica_prepare.clone(), - )); - util.leader_send(msg_from_validator_1.clone()); - - //Validate both are present in the inbound_pipe - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_validator_0 - ); - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_validator_1 - ); - - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_sanity() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.new_leader_commit(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_sanity_yield_leader_commit() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let replica_commit = util.new_replica_commit(ctx).await; - let leader_commit = util - .process_replica_commit(ctx, util.sign(replica_commit.clone())) - .await - .unwrap() - .unwrap(); - assert_eq!( - leader_commit.msg.justification, - util.new_commit_qc(|msg| *msg = replica_commit) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_bad_chain() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_commit = util.new_replica_commit(ctx).await; - replica_commit.view.genesis = rng.gen(); - let res = util - .process_replica_commit(ctx, util.sign(replica_commit)) - .await; - assert_matches!( - res, - Err(replica_commit::Error::InvalidMessage( - validator::ReplicaCommitVerifyError::View(_) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_non_validator_signer() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let replica_commit = util.new_replica_commit(ctx).await; - let non_validator_key: validator::SecretKey = ctx.rng().gen(); - let res = util - .process_replica_commit(ctx, non_validator_key.sign_msg(replica_commit)) - .await; - assert_matches!( - res, - Err(replica_commit::Error::NonValidatorSigner { signer }) => { - assert_eq!(*signer, non_validator_key.public()); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_old() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_commit = util.new_replica_commit(ctx).await; - replica_commit.view.number = ViewNumber(util.replica.view.0 - 1); - let replica_commit = util.sign(replica_commit); - let res = util.process_replica_commit(ctx, replica_commit).await; - assert_matches!( - res, - Err(replica_commit::Error::Old { current_view, current_phase }) => { - assert_eq!(current_view, util.replica.view); - assert_eq!(current_phase, util.replica.phase); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_not_leader_in_view() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let current_view_leader = util.view_leader(util.replica.view); - assert_ne!(current_view_leader, util.owner_key().public()); - let replica_commit = util.new_current_replica_commit(); - let res = util - .process_replica_commit(ctx, util.sign(replica_commit)) - .await; - assert_matches!(res, Err(replica_commit::Error::NotLeaderInView)); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_already_exists() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let replica_commit = util.new_replica_commit(ctx).await; - assert!(util - .process_replica_commit(ctx, util.sign(replica_commit.clone())) - .await - .unwrap() - .is_none()); - - // Processing twice same ReplicaCommit for same view gets DuplicateSignature error - let res = util - .process_replica_commit(ctx, util.sign(replica_commit.clone())) - .await; - assert_matches!(res, Err(replica_commit::Error::Old { .. })); - - // Processing twice different ReplicaCommit for same view gets DuplicateSignature error too - let mut different_replica_commit = replica_commit.clone(); - different_replica_commit.proposal.number = replica_commit.proposal.number.next(); - let res = util - .process_replica_commit(ctx, util.sign(different_replica_commit.clone())) - .await; - assert_matches!(res, Err(replica_commit::Error::Old { .. })); - - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_num_received_below_threshold() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - assert!(util - .process_replica_prepare(ctx, util.sign(replica_prepare.clone())) - .await - .unwrap() - .is_none()); - let replica_prepare = util.keys[1].sign_msg(replica_prepare); - let leader_prepare = util - .process_replica_prepare(ctx, replica_prepare) - .await - .unwrap() - .unwrap(); - let replica_commit = util - .process_leader_prepare(ctx, leader_prepare) - .await - .unwrap(); - util.process_replica_commit(ctx, replica_commit.clone()) - .await - .unwrap(); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_invalid_sig() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let msg = util.new_replica_commit(ctx).await; - let mut replica_commit = util.sign(msg); - replica_commit.sig = ctx.rng().gen(); - let res = util.process_replica_commit(ctx, replica_commit).await; - assert_matches!(res, Err(replica_commit::Error::InvalidSignature(..))); - Ok(()) - }) - .await - .unwrap(); -} - -/// ReplicaCommit received before sending out LeaderPrepare. -/// Whether leader accepts the message or rejects doesn't matter. -/// It just shouldn't crash. -#[tokio::test] -async fn replica_commit_unexpected_proposal() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - util.produce_block(ctx).await; - let replica_commit = util.new_current_replica_commit(); - let _ = util - .process_replica_commit(ctx, util.sign(replica_commit)) - .await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Proposal should be the same for every ReplicaCommit -/// Check it doesn't fail if one validator sends a different proposal in -/// the ReplicaCommit -#[tokio::test] -async fn replica_commit_different_proposals() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - let replica_commit = util.new_replica_commit(ctx).await; - - // Process a modified replica_commit (ie. from a malicious or wrong node) - let mut bad_replica_commit = replica_commit.clone(); - bad_replica_commit.proposal.number = replica_commit.proposal.number.next(); - util.process_replica_commit(ctx, util.sign(bad_replica_commit)) - .await - .unwrap(); - - // The rest of the validators sign the correct one - let mut replica_commit_result = None; - for i in 1..util.keys.len() { - replica_commit_result = util - .process_replica_commit(ctx, util.keys[i].sign_msg(replica_commit.clone())) - .await - .unwrap(); - } - - // Check correct proposal has been committed - assert_matches!( - replica_commit_result, - Some(leader_commit) => { - assert_eq!( - leader_commit.msg.justification.message.proposal, - replica_commit.proposal - ); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -/// Check that leader won't accumulate undefined amount of messages if -/// it's spammed with ReplicaCommit messages for future views -#[tokio::test] -async fn replica_commit_limit_messages_in_memory() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let mut replica_commit = util.new_replica_commit(ctx).await; - let mut view = util.replica_view(); - // Spam it with 200 messages for different views - for _ in 0..200 { - replica_commit.view = view.clone(); - let res = util - .process_replica_commit(ctx, util.sign(replica_commit.clone())) - .await; - assert_matches!(res, Ok(_)); - // Since we have 2 replicas, we have to send only even numbered views - // to hit the same leader (the other replica will be leader on odd numbered views) - view.number = view.number.next().next(); - } - // Ensure only 1 commit_qc is in memory, as the previous 199 were discarded each time - // new message is processed - assert_eq!(util.leader.commit_qcs.len(), 1); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn replica_commit_filter_functions_test() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let replica_commit = util.new_replica_commit(ctx).await; - let msg = util.sign(validator::ConsensusMsg::ReplicaCommit( - replica_commit.clone(), - )); - - // Send a msg with invalid signature - let mut invalid_msg = msg.clone(); - invalid_msg.sig = ctx.rng().gen(); - util.leader_send(invalid_msg); - - // Send a correct message - util.leader_send(msg.clone()); - - // Validate only correct message is received - assert_eq!(util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, msg); - - // Send a msg with view number = 2 - let mut replica_commit_from_view_2 = replica_commit.clone(); - replica_commit_from_view_2.view.number = ViewNumber(2); - let msg_from_view_2 = util.sign(validator::ConsensusMsg::ReplicaCommit( - replica_commit_from_view_2, - )); - util.leader_send(msg_from_view_2); - - // Send a msg with view number = 4, will prune message from view 2 - let mut replica_commit_from_view_4 = replica_commit.clone(); - replica_commit_from_view_4.view.number = ViewNumber(4); - let msg_from_view_4 = util.sign(validator::ConsensusMsg::ReplicaCommit( - replica_commit_from_view_4, - )); - util.leader_send(msg_from_view_4.clone()); - - // Send a msg with view number = 3, will be discarded, as it is older than message from view 4 - let mut replica_commit_from_view_3 = replica_commit.clone(); - replica_commit_from_view_3.view.number = ViewNumber(3); - let msg_from_view_3 = util.sign(validator::ConsensusMsg::ReplicaCommit( - replica_commit_from_view_3, - )); - util.leader_send(msg_from_view_3); - - // Validate only message from view 4 is received - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_view_4 - ); - - // Send a msg from validator 0 - let msg_from_validator_0 = util.keys[0].sign_msg(validator::ConsensusMsg::ReplicaCommit( - replica_commit.clone(), - )); - util.leader_send(msg_from_validator_0.clone()); - - // Send a msg from validator 1 - let msg_from_validator_1 = util.keys[1].sign_msg(validator::ConsensusMsg::ReplicaCommit( - replica_commit.clone(), - )); - util.leader_send(msg_from_validator_1.clone()); - - //Validate both are present in the inbound_pipe - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_validator_0 - ); - assert_eq!( - util.leader.inbound_pipe.recv(ctx).await.unwrap().msg, - msg_from_validator_1 - ); - - Ok(()) - }) - .await - .unwrap(); -} diff --git a/node/actors/bft/src/lib.rs b/node/actors/bft/src/lib.rs index 89061564..0075b0df 100644 --- a/node/actors/bft/src/lib.rs +++ b/node/actors/bft/src/lib.rs @@ -1,35 +1,20 @@ -//! # Consensus -//! This crate implements the Fastest-HotStuff algorithm that is described in an upcoming paper -//! It is a two-phase unchained consensus with quadratic view change (in number of authenticators, in number of -//! messages it is linear) and optimistic responsiveness. -//! -//! ## Node set -//! Right now, we assume that we have a static node set. In other words, we are running in proof-of-authority. When this repo is updated -//! to proof-of-stake, we will have a dynamic node set. -//! -//! ## Resources -//! - [Fast-HotStuff paper](https://arxiv.org/pdf/2010.11454.pdf) -//! - [HotStuff paper](https://arxiv.org/pdf/1803.05069.pdf) -//! - [HotStuff-2 paper](https://eprint.iacr.org/2023/397.pdf) -//! - [Notes on modern consensus algorithms](https://timroughgarden.github.io/fob21/andy.pdf) -//! - [Blog post comparing several consensus algorithms](https://decentralizedthoughts.github.io/2023-04-01-hotstuff-2/) -//! - Blog posts explaining [safety](https://seafooler.com/2022/01/24/understanding-safety-hotstuff/) and [responsiveness](https://seafooler.com/2022/04/02/understanding-responsiveness-hotstuff/) +//! This crate contains the consensus actor, which is responsible for handling the logic that allows us to reach agreement on blocks. +//! It uses a new cosnensus algorithm developed at Matter Labs, called ChonkyBFT. You can find the specification of the algorithm [here](../../../../spec). use crate::io::{InputMessage, OutputMessage}; use anyhow::Context; pub use config::Config; use std::sync::Arc; use tracing::Instrument; -use zksync_concurrency::{ctx, error::Wrap as _, oneshot, scope}; -use zksync_consensus_network::io::ConsensusReq; +use zksync_concurrency::{ctx, error::Wrap as _, scope, sync}; use zksync_consensus_roles::validator; use zksync_consensus_utils::pipe::ActorPipe; +/// This module contains the implementation of the ChonkyBFT algorithm. +mod chonky_bft; mod config; pub mod io; -mod leader; mod metrics; -mod replica; pub mod testonly; #[cfg(test)] mod tests; @@ -55,9 +40,6 @@ pub trait PayloadManager: std::fmt::Debug + Send + Sync { ) -> ctx::Result<()>; } -/// Channel through which bft actor sends network messages. -pub(crate) type OutputSender = ctx::channel::UnboundedSender; - impl Config { /// Starts the bft actor. It will start running, processing incoming messages and /// sending output messages. @@ -71,56 +53,39 @@ impl Config { genesis.verify().context("genesis().verify()")?; if let Some(prev) = genesis.first_block.prev() { - tracing::info!("Waiting for the pre-genesis blocks to be persisted"); + tracing::info!("Waiting for the pre-fork blocks to be persisted"); if let Err(ctx::Canceled) = self.block_store.wait_until_persisted(ctx, prev).await { return Ok(()); } } let cfg = Arc::new(self); - let (leader, leader_send) = leader::StateMachine::new(ctx, cfg.clone(), pipe.send.clone()); + let (proposer_sender, proposer_receiver) = sync::watch::channel(None); let (replica, replica_send) = - replica::StateMachine::start(ctx, cfg.clone(), pipe.send.clone()).await?; + chonky_bft::StateMachine::start(ctx, cfg.clone(), pipe.send.clone(), proposer_sender) + .await?; let res = scope::run!(ctx, |ctx, s| async { - let prepare_qc_recv = leader.prepare_qc.subscribe(); - s.spawn_bg(async { replica.run(ctx).await.wrap("replica.run()") }); - s.spawn_bg(async { leader.run(ctx).await.wrap("leader.run()") }); s.spawn_bg(async { - leader::StateMachine::run_proposer(ctx, &cfg, prepare_qc_recv, &pipe.send) + chonky_bft::proposer::run_proposer(ctx, cfg.clone(), pipe.send, proposer_receiver) .await .wrap("run_proposer()") }); tracing::info!("Starting consensus actor {:?}", cfg.secret_key.public()); - // This is the infinite loop where the consensus actually runs. The validator waits for either - // a message from the network or for a timeout, and processes each accordingly. + // This is the infinite loop where the consensus actually runs. The validator waits for + // a message from the network and processes it accordingly. loop { async { - let InputMessage::Network(req) = pipe + let InputMessage::Network(msg) = pipe .recv .recv(ctx) .instrument(tracing::info_span!("wait_for_message")) .await?; - use validator::ConsensusMsg as M; - match &req.msg.msg { - M::ReplicaPrepare(_) => { - // This is a hacky way to do a clone. This is necessary since we don't want to derive - // Clone for ConsensusReq. When we change to ChonkyBFT this will be removed anyway. - let (ack, _) = oneshot::channel(); - let new_req = ConsensusReq { - msg: req.msg.clone(), - ack, - }; - replica_send.send(new_req); - leader_send.send(req); - } - M::ReplicaCommit(_) => leader_send.send(req), - M::LeaderPrepare(_) | M::LeaderCommit(_) => replica_send.send(req), - } + replica_send.send(msg); ctx::Ok(()) } diff --git a/node/actors/bft/src/metrics.rs b/node/actors/bft/src/metrics.rs index ea23c711..248b57c9 100644 --- a/node/actors/bft/src/metrics.rs +++ b/node/actors/bft/src/metrics.rs @@ -12,14 +12,14 @@ const PAYLOAD_SIZE_BUCKETS: Buckets = Buckets::exponential( #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] #[metrics(rename_all = "snake_case")] pub(crate) enum ConsensusMsgLabel { - /// Label for a `LeaderPrepare` message. - LeaderPrepare, - /// Label for a `LeaderCommit` message. - LeaderCommit, - /// Label for a `ReplicaPrepare` message. - ReplicaPrepare, + /// Label for a `LeaderProposal` message. + LeaderProposal, /// Label for a `ReplicaCommit` message. ReplicaCommit, + /// Label for a `ReplicaTimeout` message. + ReplicaTimeout, + /// Label for a `ReplicaNewView` message. + ReplicaNewView, } impl ConsensusMsgLabel { @@ -53,26 +53,29 @@ pub(crate) struct ProcessingLatencyLabels { #[derive(Debug, Metrics)] #[metrics(prefix = "consensus")] pub(crate) struct ConsensusMetrics { + /// Number of the current view of the replica. + #[metrics(unit = Unit::Seconds)] + pub(crate) replica_view_number: Gauge, + /// Number of the last finalized block observed by the node. + pub(crate) finalized_block_number: Gauge, /// Size of the proposed payload in bytes. #[metrics(buckets = PAYLOAD_SIZE_BUCKETS, unit = Unit::Bytes)] - pub(crate) leader_proposal_payload_size: Histogram, - /// Latency of the commit phase observed by the leader. + pub(crate) proposal_payload_size: Histogram, + /// Latency of receiving a proposal as observed by the replica. Measures from + /// the start of the view until we have a verified proposal. #[metrics(buckets = Buckets::exponential(0.01..=20.0, 1.5), unit = Unit::Seconds)] - pub(crate) leader_commit_phase_latency: Histogram, - /// Currently set timeout after which replica will proceed to the next view. - #[metrics(unit = Unit::Seconds)] - pub(crate) replica_view_timeout: Gauge, + pub(crate) proposal_latency: Histogram, + /// Latency of committing to a block as observed by the replica. Measures from + /// the start of the view until we send a commit vote. + #[metrics(buckets = Buckets::exponential(0.01..=20.0, 1.5), unit = Unit::Seconds)] + pub(crate) commit_latency: Histogram, + /// Latency of a single view as observed by the replica. Measures from + /// the start of the view until the start of the next. + #[metrics(buckets = Buckets::exponential(0.01..=20.0, 1.5), unit = Unit::Seconds)] + pub(crate) view_latency: Histogram, /// Latency of processing messages by the replicas. #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)] - pub(crate) replica_processing_latency: Family>, - /// Latency of processing messages by the leader. - #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)] - pub(crate) leader_processing_latency: Family>, - /// Number of the last finalized block observed by the node. - pub(crate) finalized_block_number: Gauge, - /// Number of the current view of the replica. - #[metrics(unit = Unit::Seconds)] - pub(crate) replica_view_number: Gauge, + pub(crate) message_processing_latency: Family>, } /// Global instance of [`ConsensusMetrics`]. diff --git a/node/actors/bft/src/replica/leader_commit.rs b/node/actors/bft/src/replica/leader_commit.rs deleted file mode 100644 index d60b99b3..00000000 --- a/node/actors/bft/src/replica/leader_commit.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Handler of a LeaderCommit message. -use super::StateMachine; -use zksync_concurrency::{ctx, error::Wrap}; -use zksync_consensus_roles::validator; - -/// Errors that can occur when processing a "leader commit" message. -#[derive(Debug, thiserror::Error)] -pub(crate) enum Error { - /// Invalid leader. - #[error("bad leader: got {got:?}, want {want:?}")] - BadLeader { - /// Received leader. - got: validator::PublicKey, - /// Correct leader. - want: validator::PublicKey, - }, - /// Past view of phase. - #[error("past view/phase (current view: {current_view:?}, current phase: {current_phase:?})")] - Old { - /// Current view. - current_view: validator::ViewNumber, - /// Current phase. - current_phase: validator::Phase, - }, - /// Invalid message signature. - #[error("invalid signature: {0:#}")] - InvalidSignature(#[source] anyhow::Error), - /// Invalid message. - #[error("invalid message: {0:#}")] - InvalidMessage(validator::CommitQCVerifyError), - /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. - #[error(transparent)] - Internal(#[from] ctx::Error), -} - -impl Wrap for Error { - fn with_wrap C>( - self, - f: F, - ) -> Self { - match self { - Error::Internal(err) => Error::Internal(err.with_wrap(f)), - err => err, - } - } -} - -impl StateMachine { - /// Processes a leader commit message. We can approve this leader message even if we - /// don't have the block proposal stored. It is enough to see the justification. - pub(crate) async fn process_leader_commit( - &mut self, - ctx: &ctx::Ctx, - signed_message: validator::Signed, - ) -> Result<(), Error> { - // ----------- Checking origin of the message -------------- - - // Unwrap message. - let message = &signed_message.msg; - let author = &signed_message.key; - - // Check that it comes from the correct leader. - let leader = self.config.genesis().view_leader(message.view().number); - if author != &leader { - return Err(Error::BadLeader { - want: leader, - got: author.clone(), - }); - } - - // If the message is from the "past", we discard it. - if (message.view().number, validator::Phase::Commit) < (self.view, self.phase) { - return Err(Error::Old { - current_view: self.view, - current_phase: self.phase, - }); - } - - // ----------- Checking the signed part of the message -------------- - - // Check the signature on the message. - signed_message.verify().map_err(Error::InvalidSignature)?; - message - .verify(self.config.genesis()) - .map_err(Error::InvalidMessage)?; - - // ----------- All checks finished. Now we process the message. -------------- - - // Try to create a finalized block with this CommitQC and our block proposal cache. - self.save_block(ctx, &message.justification) - .await - .wrap("save_block()")?; - - // Start a new view. But first we skip to the view of this message. - self.view = message.view().number; - self.start_new_view(ctx).await.wrap("start_new_view()")?; - - Ok(()) - } -} diff --git a/node/actors/bft/src/replica/leader_prepare.rs b/node/actors/bft/src/replica/leader_prepare.rs deleted file mode 100644 index 55e43cf0..00000000 --- a/node/actors/bft/src/replica/leader_prepare.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Handler of a LeaderPrepare message. -use super::StateMachine; -use zksync_concurrency::{ctx, error::Wrap}; -use zksync_consensus_network::io::{ConsensusInputMessage, Target}; -use zksync_consensus_roles::validator::{self, BlockNumber}; - -/// Errors that can occur when processing a "leader prepare" message. -#[derive(Debug, thiserror::Error)] -pub(crate) enum Error { - /// Invalid leader. - #[error( - "invalid leader (correct leader: {correct_leader:?}, received leader: {received_leader:?})" - )] - InvalidLeader { - /// Correct leader. - correct_leader: validator::PublicKey, - /// Received leader. - received_leader: validator::PublicKey, - }, - /// Message for a past view or phase. - #[error( - "message for a past view / phase (current view: {current_view:?}, current phase: {current_phase:?})" - )] - Old { - /// Current view. - current_view: validator::ViewNumber, - /// Current phase. - current_phase: validator::Phase, - }, - /// Invalid message signature. - #[error("invalid signature: {0:#}")] - InvalidSignature(#[source] anyhow::Error), - /// Invalid message. - #[error("invalid message: {0:#}")] - InvalidMessage(#[source] validator::LeaderPrepareVerifyError), - /// Leader proposed a block that was already pruned from replica's storage. - #[error("leader proposed a block that was already pruned from replica's storage")] - ProposalAlreadyPruned, - /// Oversized payload. - #[error("block proposal with an oversized payload (payload size: {payload_size})")] - ProposalOversizedPayload { - /// Size of the payload. - payload_size: usize, - }, - /// Invalid payload. - #[error("invalid payload: {0:#}")] - ProposalInvalidPayload(#[source] anyhow::Error), - /// Previous payload missing. - #[error("previous block proposal payload missing from store (block number: {prev_number})")] - MissingPreviousPayload { - /// The number of the missing block - prev_number: BlockNumber, - }, - /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. - #[error(transparent)] - Internal(#[from] ctx::Error), -} - -impl Wrap for Error { - fn with_wrap C>( - self, - f: F, - ) -> Self { - match self { - Error::Internal(err) => Error::Internal(err.with_wrap(f)), - err => err, - } - } -} - -impl StateMachine { - /// Processes a leader prepare message. - pub(crate) async fn process_leader_prepare( - &mut self, - ctx: &ctx::Ctx, - signed_message: validator::Signed, - ) -> Result<(), Error> { - // ----------- Checking origin of the message -------------- - - // Unwrap message. - let message = &signed_message.msg; - let author = &signed_message.key; - let view = message.view().number; - - // Check that it comes from the correct leader. - let leader = self.config.genesis().view_leader(view); - if author != &leader { - return Err(Error::InvalidLeader { - correct_leader: leader, - received_leader: author.clone(), - }); - } - - // If the message is from the "past", we discard it. - if (view, validator::Phase::Prepare) < (self.view, self.phase) { - return Err(Error::Old { - current_view: self.view, - current_phase: self.phase, - }); - } - - // Replica MUSTN'T vote for blocks which have been already pruned for storage. - // (because it won't be able to persist and broadcast them once finalized). - // TODO(gprusak): it should never happen, we should add safety checks to prevent - // pruning blocks not known to be finalized. - if message.proposal.number < self.config.block_store.queued().first { - return Err(Error::ProposalAlreadyPruned); - } - - // ----------- Checking the message -------------- - - signed_message.verify().map_err(Error::InvalidSignature)?; - message - .verify(self.config.genesis()) - .map_err(Error::InvalidMessage)?; - - let high_qc = message.justification.high_qc(); - - if let Some(high_qc) = high_qc { - // Try to create a finalized block with this CommitQC and our block proposal cache. - // This gives us another chance to finalize a block that we may have missed before. - self.save_block(ctx, high_qc).await.wrap("save_block()")?; - } - - // Check that the payload doesn't exceed the maximum size. - if let Some(payload) = &message.proposal_payload { - if payload.0.len() > self.config.max_payload_size { - return Err(Error::ProposalOversizedPayload { - payload_size: payload.0.len(), - }); - } - - if let Some(prev) = message.proposal.number.prev() { - // Defensively assume that PayloadManager cannot verify proposal until the previous block is stored. - self.config - .block_store - .wait_until_persisted(&ctx.with_deadline(self.timeout_deadline), prev) - .await - .map_err(|_| Error::MissingPreviousPayload { prev_number: prev })?; - } - if let Err(err) = self - .config - .payload_manager - .verify(ctx, message.proposal.number, payload) - .await - { - return Err(match err { - err @ ctx::Error::Canceled(_) => Error::Internal(err), - ctx::Error::Internal(err) => Error::ProposalInvalidPayload(err), - }); - } - } - - // ----------- All checks finished. Now we process the message. -------------- - - // Create our commit vote. - let commit_vote = validator::ReplicaCommit { - view: message.view().clone(), - proposal: message.proposal, - }; - - // Update the state machine. - self.view = message.view().number; - self.phase = validator::Phase::Commit; - self.high_vote = Some(commit_vote.clone()); - // If we received a new block proposal, store it in our cache. - if let Some(payload) = &message.proposal_payload { - self.block_proposal_cache - .entry(message.proposal.number) - .or_default() - .insert(payload.hash(), payload.clone()); - } - - // Backup our state. - self.backup_state(ctx).await.wrap("backup_state()")?; - - // Send the replica message to the leader. - let output_message = ConsensusInputMessage { - message: self - .config - .secret_key - .sign_msg(validator::ConsensusMsg::ReplicaCommit(commit_vote)), - recipient: Target::Validator(author.clone()), - }; - self.outbound_pipe.send(output_message.into()); - - Ok(()) - } -} diff --git a/node/actors/bft/src/replica/mod.rs b/node/actors/bft/src/replica/mod.rs deleted file mode 100644 index 640f044b..00000000 --- a/node/actors/bft/src/replica/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Implements the replica role in the Fastest-HotStuff consensus algorithm. The replica is the role that validates -//! proposals, votes for them and finalizes them. It basically drives the consensus forward. Note that our consensus -//! node will perform both the replica and leader roles simultaneously. - -mod block; -pub(crate) mod leader_commit; -pub(crate) mod leader_prepare; -mod new_view; -pub(crate) mod replica_prepare; -mod state_machine; -#[cfg(test)] -mod tests; -mod timer; - -pub(crate) use self::state_machine::StateMachine; diff --git a/node/actors/bft/src/replica/new_view.rs b/node/actors/bft/src/replica/new_view.rs deleted file mode 100644 index 16403136..00000000 --- a/node/actors/bft/src/replica/new_view.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::StateMachine; -use crate::metrics; -use zksync_concurrency::{ctx, error::Wrap as _}; -use zksync_consensus_network::io::{ConsensusInputMessage, Target}; -use zksync_consensus_roles::validator; - -impl StateMachine { - /// This blocking method is used whenever we start a new view. - pub(crate) async fn start_new_view(&mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { - // Update the state machine. - self.view = self.view.next(); - tracing::info!("Starting view {}", self.view); - metrics::METRICS.replica_view_number.set(self.view.0); - - self.phase = validator::Phase::Prepare; - if let Some(qc) = self.high_qc.as_ref() { - // Clear the block cache. - self.block_proposal_cache - .retain(|k, _| k > &qc.header().number); - } - - // Backup our state. - self.backup_state(ctx).await.wrap("backup_state()")?; - - // Send the replica message. - let output_message = ConsensusInputMessage { - message: self - .config - .secret_key - .sign_msg(validator::ConsensusMsg::ReplicaPrepare( - validator::ReplicaPrepare { - view: validator::View { - genesis: self.config.genesis().hash(), - number: self.view, - }, - high_vote: self.high_vote.clone(), - high_qc: self.high_qc.clone(), - }, - )), - recipient: Target::Broadcast, - }; - self.outbound_pipe.send(output_message.into()); - - // Reset the timer. - self.reset_timer(ctx); - Ok(()) - } -} diff --git a/node/actors/bft/src/replica/replica_prepare.rs b/node/actors/bft/src/replica/replica_prepare.rs deleted file mode 100644 index 74b09ad9..00000000 --- a/node/actors/bft/src/replica/replica_prepare.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! Handler of a ReplicaPrepare message. -use super::StateMachine; -use zksync_concurrency::{ctx, error::Wrap}; -use zksync_consensus_roles::validator; - -/// Errors that can occur when processing a "replica prepare" message. -#[derive(Debug, thiserror::Error)] -pub(crate) enum Error { - /// Message signer isn't part of the validator set. - #[error("Message signer isn't part of the validator set (signer: {signer:?})")] - NonValidatorSigner { - /// Signer of the message. - signer: validator::PublicKey, - }, - /// Past view or phase. - #[error("past view/phase (current view: {current_view:?}, current phase: {current_phase:?})")] - Old { - /// Current view. - current_view: validator::ViewNumber, - /// Current phase. - current_phase: validator::Phase, - }, - /// Invalid message signature. - #[error("invalid signature: {0:#}")] - InvalidSignature(#[source] anyhow::Error), - /// Invalid message. - #[error(transparent)] - InvalidMessage(validator::ReplicaPrepareVerifyError), - /// Internal error. Unlike other error types, this one isn't supposed to be easily recoverable. - #[error(transparent)] - Internal(#[from] ctx::Error), -} - -impl Wrap for Error { - fn with_wrap C>( - self, - f: F, - ) -> Self { - match self { - Error::Internal(err) => Error::Internal(err.with_wrap(f)), - err => err, - } - } -} - -impl StateMachine { - /// Processes `ReplicaPrepare` message. - pub(crate) async fn process_replica_prepare( - &mut self, - ctx: &ctx::Ctx, - signed_message: validator::Signed, - ) -> Result<(), Error> { - // ----------- Checking origin of the message -------------- - - // Unwrap message. - let message = signed_message.msg.clone(); - let author = &signed_message.key; - - // Check that the message signer is in the validator set. - if !self.config.genesis().validators.contains(author) { - return Err(Error::NonValidatorSigner { - signer: author.clone(), - }); - } - - // We only accept this type of message from the future. - if message.view.number <= self.view { - return Err(Error::Old { - current_view: self.view, - current_phase: self.phase, - }); - } - - // ----------- Checking the signed part of the message -------------- - - // Check the signature on the message. - signed_message.verify().map_err(Error::InvalidSignature)?; - - // Extract the QC and verify it. - let Some(high_qc) = message.high_qc else { - return Ok(()); - }; - - high_qc.verify(self.config.genesis()).map_err(|err| { - Error::InvalidMessage(validator::ReplicaPrepareVerifyError::HighQC(err)) - })?; - - // ----------- All checks finished. Now we process the message. -------------- - - let qc_view = high_qc.view().number; - - // Try to create a finalized block with this CommitQC and our block proposal cache. - // It will also update our high QC, if necessary. - self.save_block(ctx, &high_qc).await.wrap("save_block()")?; - - // Skip to a new view, if necessary. - if qc_view >= self.view { - self.view = qc_view; - self.start_new_view(ctx).await.wrap("start_new_view()")?; - } - - Ok(()) - } -} diff --git a/node/actors/bft/src/replica/state_machine.rs b/node/actors/bft/src/replica/state_machine.rs deleted file mode 100644 index 238cc0e3..00000000 --- a/node/actors/bft/src/replica/state_machine.rs +++ /dev/null @@ -1,255 +0,0 @@ -use crate::{metrics, Config, OutputSender}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; -use zksync_concurrency::{ - ctx, - error::Wrap as _, - metrics::LatencyHistogramExt as _, - sync::{self, prunable_mpsc::SelectionFunctionResult}, - time, -}; -use zksync_consensus_network::io::ConsensusReq; -use zksync_consensus_roles::{validator, validator::ConsensusMsg}; -use zksync_consensus_storage as storage; - -/// The StateMachine struct contains the state of the replica. This is the most complex state machine and is responsible -/// for validating and voting on blocks. When participating in consensus we are always a replica. -#[derive(Debug)] -pub(crate) struct StateMachine { - /// Consensus configuration and output channel. - pub(crate) config: Arc, - /// Pipe through which replica sends network messages. - pub(super) outbound_pipe: OutputSender, - /// Pipe through which replica receives network requests. - inbound_pipe: sync::prunable_mpsc::Receiver, - /// The current view number. - pub(crate) view: validator::ViewNumber, - /// The current phase. - pub(crate) phase: validator::Phase, - /// The highest block proposal that the replica has committed to. - pub(crate) high_vote: Option, - /// The highest commit quorum certificate known to the replica. - pub(crate) high_qc: Option, - /// A cache of the received block proposals. - pub(crate) block_proposal_cache: - BTreeMap>, - /// The deadline to receive an input message. - pub(crate) timeout_deadline: time::Deadline, -} - -impl StateMachine { - /// Creates a new [`StateMachine`] instance, attempting to recover a past state from the storage module, - /// otherwise initializes the state machine with the current head block. - /// - /// Returns a tuple containing: - /// * The newly created [`StateMachine`] instance. - /// * A sender handle that should be used to send values to be processed by the instance, asynchronously. - pub(crate) async fn start( - ctx: &ctx::Ctx, - config: Arc, - outbound_pipe: OutputSender, - ) -> ctx::Result<(Self, sync::prunable_mpsc::Sender)> { - let backup = config.replica_store.state(ctx).await?; - let mut block_proposal_cache: BTreeMap<_, HashMap<_, _>> = BTreeMap::new(); - for proposal in backup.proposals { - block_proposal_cache - .entry(proposal.number) - .or_default() - .insert(proposal.payload.hash(), proposal.payload); - } - - let (send, recv) = sync::prunable_mpsc::channel( - StateMachine::inbound_filter_predicate, - StateMachine::inbound_selection_function, - ); - - let mut this = Self { - config, - outbound_pipe, - inbound_pipe: recv, - view: backup.view, - phase: backup.phase, - high_vote: backup.high_vote, - high_qc: backup.high_qc, - block_proposal_cache, - timeout_deadline: time::Deadline::Infinite, - }; - - // We need to start the replica before processing inputs. - this.start_new_view(ctx).await.wrap("start_new_view()")?; - - Ok((this, send)) - } - - /// Runs a loop to process incoming messages (may be `None` if the channel times out while waiting for a message). - /// This is the main entry point for the state machine, - /// potentially triggering state modifications and message sending to the executor. - pub(crate) async fn run(mut self, ctx: &ctx::Ctx) -> ctx::Result<()> { - loop { - let recv = self - .inbound_pipe - .recv(&ctx.with_deadline(self.timeout_deadline)) - .await; - - // Check for non-timeout cancellation. - if !ctx.is_active() { - return Ok(()); - } - - // Check for timeout. - let Some(req) = recv.ok() else { - self.start_new_view(ctx).await?; - continue; - }; - - let now = ctx.now(); - let label = match &req.msg.msg { - ConsensusMsg::ReplicaPrepare(_) => { - let res = match self - .process_replica_prepare(ctx, req.msg.cast().unwrap()) - .await - .wrap("process_replica_prepare()") - { - Ok(()) => Ok(()), - Err(err) => { - match err { - super::replica_prepare::Error::Internal(e) => { - tracing::error!( - "process_replica_prepare: internal error: {e:#}" - ); - return Err(e); - } - super::replica_prepare::Error::Old { .. } => { - tracing::debug!("process_replica_prepare: {err:#}"); - } - _ => { - tracing::warn!("process_replica_prepare: {err:#}"); - } - } - Err(()) - } - }; - metrics::ConsensusMsgLabel::ReplicaPrepare.with_result(&res) - } - ConsensusMsg::LeaderPrepare(_) => { - let res = match self - .process_leader_prepare(ctx, req.msg.cast().unwrap()) - .await - .wrap("process_leader_prepare()") - { - Ok(()) => Ok(()), - Err(err) => { - match err { - super::leader_prepare::Error::Internal(e) => { - tracing::error!( - "process_leader_prepare: internal error: {e:#}" - ); - return Err(e); - } - super::leader_prepare::Error::Old { .. } => { - tracing::info!("process_leader_prepare: {err:#}"); - } - _ => { - tracing::warn!("process_leader_prepare: {err:#}"); - } - } - Err(()) - } - }; - metrics::ConsensusMsgLabel::LeaderPrepare.with_result(&res) - } - ConsensusMsg::LeaderCommit(_) => { - let res = match self - .process_leader_commit(ctx, req.msg.cast().unwrap()) - .await - .wrap("process_leader_commit()") - { - Ok(()) => Ok(()), - Err(err) => { - match err { - super::leader_commit::Error::Internal(e) => { - tracing::error!("process_leader_commit: internal error: {e:#}"); - return Err(e); - } - super::leader_commit::Error::Old { .. } => { - tracing::info!("process_leader_commit: {err:#}"); - } - _ => { - tracing::warn!("process_leader_commit: {err:#}"); - } - } - Err(()) - } - }; - metrics::ConsensusMsgLabel::LeaderCommit.with_result(&res) - } - _ => unreachable!(), - }; - metrics::METRICS.replica_processing_latency[&label].observe_latency(ctx.now() - now); - - // Notify network actor that the message has been processed. - // Ignore sending error. - let _ = req.ack.send(()); - } - } - - /// Backups the replica state to disk. - pub(crate) async fn backup_state(&self, ctx: &ctx::Ctx) -> ctx::Result<()> { - let mut proposals = vec![]; - for (number, payloads) in &self.block_proposal_cache { - proposals.extend(payloads.values().map(|p| storage::Proposal { - number: *number, - payload: p.clone(), - })); - } - let backup = storage::ReplicaState { - view: self.view, - phase: self.phase, - high_vote: self.high_vote.clone(), - high_qc: self.high_qc.clone(), - proposals, - }; - self.config - .replica_store - .set_state(ctx, &backup) - .await - .wrap("put_replica_state")?; - Ok(()) - } - - fn inbound_filter_predicate(new_req: &ConsensusReq) -> bool { - // Verify message signature - new_req.msg.verify().is_ok() - } - - fn inbound_selection_function( - old_req: &ConsensusReq, - new_req: &ConsensusReq, - ) -> SelectionFunctionResult { - if old_req.msg.key != new_req.msg.key { - return SelectionFunctionResult::Keep; - } - - match (&old_req.msg.msg, &new_req.msg.msg) { - (ConsensusMsg::LeaderPrepare(old), ConsensusMsg::LeaderPrepare(new)) => { - // Discard older message - if old.view().number < new.view().number { - SelectionFunctionResult::DiscardOld - } else { - SelectionFunctionResult::DiscardNew - } - } - (ConsensusMsg::LeaderCommit(old), ConsensusMsg::LeaderCommit(new)) => { - // Discard older message - if old.view().number < new.view().number { - SelectionFunctionResult::DiscardOld - } else { - SelectionFunctionResult::DiscardNew - } - } - _ => SelectionFunctionResult::Keep, - } - } -} diff --git a/node/actors/bft/src/replica/tests.rs b/node/actors/bft/src/replica/tests.rs deleted file mode 100644 index 10d7407d..00000000 --- a/node/actors/bft/src/replica/tests.rs +++ /dev/null @@ -1,625 +0,0 @@ -use super::{leader_commit, leader_prepare}; -use crate::{ - testonly, - testonly::ut_harness::{UTHarness, MAX_PAYLOAD_SIZE}, -}; -use assert_matches::assert_matches; -use rand::Rng; -use zksync_concurrency::{ctx, scope}; -use zksync_consensus_roles::validator::{ - self, CommitQC, Payload, PrepareQC, ReplicaCommit, ReplicaPrepare, -}; - -/// Sanity check of the happy path. -#[tokio::test] -async fn block_production() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - util.produce_block(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Sanity check of block production with reproposal. -#[tokio::test] -async fn reproposal_block_production() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - util.new_leader_commit(ctx).await; - util.process_replica_timeout(ctx).await; - util.produce_block(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_bad_chain() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_prepare = util.new_leader_prepare(ctx).await; - leader_prepare.justification.view.genesis = rng.gen(); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::Justification( - validator::PrepareQCVerifyError::View(_) - ) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_sanity_yield_replica_commit() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let leader_prepare = util.new_leader_prepare(ctx).await; - let replica_commit = util - .process_leader_prepare(ctx, util.sign(leader_prepare.clone())) - .await - .unwrap(); - assert_eq!( - replica_commit.msg, - ReplicaCommit { - view: leader_prepare.view().clone(), - proposal: leader_prepare.proposal, - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_invalid_leader() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - - let replica_prepare = util.new_replica_prepare(); - assert!(util - .process_replica_prepare(ctx, util.sign(replica_prepare.clone())) - .await - .unwrap() - .is_none()); - - let replica_prepare = util.keys[1].sign_msg(replica_prepare); - let mut leader_prepare = util - .process_replica_prepare(ctx, replica_prepare) - .await - .unwrap() - .unwrap() - .msg; - leader_prepare.justification.view.number = leader_prepare.justification.view.number.next(); - assert_ne!( - util.view_leader(leader_prepare.view().number), - util.keys[0].public() - ); - - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidLeader { correct_leader, received_leader }) => { - assert_eq!(correct_leader, util.keys[1].public()); - assert_eq!(received_leader, util.keys[0].public()); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_old_view() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_prepare = util.new_leader_prepare(ctx).await; - leader_prepare.justification.view.number.0 = util.replica.view.0 - 1; - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::Old { current_view, current_phase }) => { - assert_eq!(current_view, util.replica.view); - assert_eq!(current_phase, util.replica.phase); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_pruned_block() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_prepare = util.new_leader_prepare(ctx).await; - // We assume default replica state and nontrivial `genesis.fork.first_block` here. - leader_prepare.proposal.number = util - .replica - .config - .block_store - .queued() - .first - .prev() - .unwrap(); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!(res, Err(leader_prepare::Error::ProposalAlreadyPruned)); - Ok(()) - }) - .await - .unwrap(); -} - -/// Tests that `WriteBlockStore::verify_payload` is applied before signing a vote. -#[tokio::test] -async fn leader_prepare_invalid_payload() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = - UTHarness::new_with_payload(ctx, 1, Box::new(testonly::RejectPayload)).await; - s.spawn_bg(runner.run(ctx)); - - let leader_prepare = util.new_leader_prepare(ctx).await; - - // Insert a finalized block to the storage. - let mut justification = CommitQC::new( - ReplicaCommit { - view: util.replica_view(), - proposal: leader_prepare.proposal, - }, - util.genesis(), - ); - justification - .add(&util.sign(justification.message.clone()), util.genesis()) - .unwrap(); - let block = validator::FinalBlock { - payload: leader_prepare.proposal_payload.clone().unwrap(), - justification, - }; - util.replica - .config - .block_store - .queue_block(ctx, block.into()) - .await - .unwrap(); - - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!(res, Err(leader_prepare::Error::ProposalInvalidPayload(..))); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_invalid_sig() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - let leader_prepare = util.new_leader_prepare(ctx).await; - let mut leader_prepare = util.sign(leader_prepare); - leader_prepare.sig = ctx.rng().gen(); - let res = util.process_leader_prepare(ctx, leader_prepare).await; - assert_matches!(res, Err(leader_prepare::Error::InvalidSignature(..))); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_invalid_prepare_qc() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_prepare = util.new_leader_prepare(ctx).await; - leader_prepare.justification.signature = ctx.rng().gen(); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::Justification(_) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_proposal_oversized_payload() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let payload_oversize = MAX_PAYLOAD_SIZE + 1; - let payload = Payload(vec![0; payload_oversize]); - let mut leader_prepare = util.new_leader_prepare(ctx).await; - leader_prepare.proposal.payload = payload.hash(); - leader_prepare.proposal_payload = Some(payload); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::ProposalOversizedPayload{ payload_size }) => { - assert_eq!(payload_size, payload_oversize); - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_proposal_mismatched_payload() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_prepare = util.new_leader_prepare(ctx).await; - leader_prepare.proposal_payload = Some(ctx.rng().gen()); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::ProposalMismatchedPayload - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_proposal_when_previous_not_finalized() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - tracing::info!("Execute view without replicas receiving the LeaderCommit."); - util.new_leader_commit(ctx).await; - util.process_replica_timeout(ctx).await; - tracing::info!("Make leader repropose the block."); - let mut leader_prepare = util.new_leader_prepare(ctx).await; - tracing::info!("Modify the message to include a new proposal anyway."); - let payload: Payload = rng.gen(); - leader_prepare.proposal.payload = payload.hash(); - leader_prepare.proposal_payload = Some(payload); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::ProposalWhenPreviousNotFinalized - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_bad_block_number() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx,s| async { - let (mut util,runner) = UTHarness::new(ctx,1).await; - s.spawn_bg(runner.run(ctx)); - - tracing::info!("Produce initial block."); - util.produce_block(ctx).await; - tracing::info!("Make leader propose the next block."); - let mut leader_prepare = util.new_leader_prepare(ctx).await; - tracing::info!("Modify the proposal.number so that it doesn't match the previous block"); - leader_prepare.proposal.number = rng.gen(); - let res = util.process_leader_prepare(ctx, util.sign(leader_prepare.clone())).await; - assert_matches!(res, Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::BadBlockNumber { got, want } - )) => { - assert_eq!(want, leader_prepare.justification.high_qc().unwrap().message.proposal.number.next()); - assert_eq!(got, leader_prepare.proposal.number); - }); - Ok(()) - }).await.unwrap(); -} - -#[tokio::test] -async fn leader_prepare_reproposal_without_quorum() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - tracing::info!("make leader repropose a block"); - util.new_leader_commit(ctx).await; - util.process_replica_timeout(ctx).await; - let mut leader_prepare = util.new_leader_prepare(ctx).await; - tracing::info!("modify justification, to make reproposal unjustified"); - let mut replica_prepare: ReplicaPrepare = leader_prepare - .justification - .map - .keys() - .next() - .unwrap() - .clone(); - leader_prepare.justification = PrepareQC::new(leader_prepare.justification.view); - for key in &util.keys { - replica_prepare.high_vote.as_mut().unwrap().proposal.payload = rng.gen(); - leader_prepare - .justification - .add(&key.sign_msg(replica_prepare.clone()), util.genesis())?; - } - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::ReproposalWithoutQuorum - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_reproposal_when_finalized() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - tracing::info!("Make leader propose a new block"); - util.produce_block(ctx).await; - let mut leader_prepare = util.new_leader_prepare(ctx).await; - tracing::info!( - "Modify the message so that it is actually a reproposal of the previous block" - ); - leader_prepare.proposal = leader_prepare - .justification - .high_qc() - .unwrap() - .message - .proposal; - leader_prepare.proposal_payload = None; - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::ReproposalWhenFinalized - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_prepare_reproposal_invalid_block() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - tracing::info!("Make leader repropose a block."); - util.new_leader_commit(ctx).await; - util.process_replica_timeout(ctx).await; - let mut leader_prepare = util.new_leader_prepare(ctx).await; - tracing::info!("Make the reproposal different than expected"); - leader_prepare.proposal.payload = rng.gen(); - let res = util - .process_leader_prepare(ctx, util.sign(leader_prepare)) - .await; - assert_matches!( - res, - Err(leader_prepare::Error::InvalidMessage( - validator::LeaderPrepareVerifyError::ReproposalBadBlock - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -/// Check that replica provides expected high_vote and high_qc after finalizing a block. -#[tokio::test] -async fn leader_commit_sanity_yield_replica_prepare() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let leader_commit = util.new_leader_commit(ctx).await; - let replica_prepare = util - .process_leader_commit(ctx, util.sign(leader_commit.clone())) - .await - .unwrap(); - let mut view = leader_commit.justification.message.view.clone(); - view.number = view.number.next(); - assert_eq!( - replica_prepare.msg, - ReplicaPrepare { - view, - high_vote: Some(leader_commit.justification.message.clone()), - high_qc: Some(leader_commit.justification), - } - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_commit_bad_chain() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_commit = util.new_leader_commit(ctx).await; - leader_commit.justification.message.view.genesis = rng.gen(); - let res = util - .process_leader_commit(ctx, util.sign(leader_commit)) - .await; - assert_matches!( - res, - Err(leader_commit::Error::InvalidMessage( - validator::CommitQCVerifyError::InvalidMessage( - validator::ReplicaCommitVerifyError::View(_) - ) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_commit_bad_leader() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 2).await; - s.spawn_bg(runner.run(ctx)); - let leader_commit = util.new_leader_commit(ctx).await; - // Sign the leader_prepare with a key of different validator. - let res = util - .process_leader_commit(ctx, util.keys[1].sign_msg(leader_commit)) - .await; - assert_matches!(res, Err(leader_commit::Error::BadLeader { .. })); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_commit_invalid_sig() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - let leader_commit = util.new_leader_commit(ctx).await; - let mut leader_commit = util.sign(leader_commit); - leader_commit.sig = rng.gen(); - let res = util.process_leader_commit(ctx, leader_commit).await; - assert_matches!(res, Err(leader_commit::Error::InvalidSignature { .. })); - Ok(()) - }) - .await - .unwrap(); -} - -#[tokio::test] -async fn leader_commit_invalid_commit_qc() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new(ctx, 1).await; - s.spawn_bg(runner.run(ctx)); - - let mut leader_commit = util.new_leader_commit(ctx).await; - leader_commit.justification.signature = rng.gen(); - let res = util - .process_leader_commit(ctx, util.sign(leader_commit)) - .await; - assert_matches!( - res, - Err(leader_commit::Error::InvalidMessage( - validator::CommitQCVerifyError::BadSignature(..) - )) - ); - Ok(()) - }) - .await - .unwrap(); -} diff --git a/node/actors/bft/src/replica/timer.rs b/node/actors/bft/src/replica/timer.rs deleted file mode 100644 index 75570d2d..00000000 --- a/node/actors/bft/src/replica/timer.rs +++ /dev/null @@ -1,35 +0,0 @@ -use super::StateMachine; -use crate::metrics; -use zksync_concurrency::{ctx, metrics::LatencyGaugeExt as _, time}; -use zksync_consensus_roles::validator; - -impl StateMachine { - /// The base duration of the timeout. - pub(crate) const BASE_DURATION: time::Duration = time::Duration::milliseconds(2000); - /// Max duration of the timeout. - /// Consensus is unusable with this range of timeout anyway, - /// however to make debugging easier we bound it to a specific value. - pub(crate) const MAX_DURATION: time::Duration = time::Duration::seconds(1000000); - - /// Resets the timer. On every timeout we double the duration, starting from a given base duration. - /// This is a simple exponential backoff. - pub(crate) fn reset_timer(&mut self, ctx: &ctx::Ctx) { - let final_view = match self.high_qc.as_ref() { - Some(qc) => qc.view().number.next(), - None => validator::ViewNumber(0), - }; - let f = self - .view - .0 - .saturating_sub(final_view.0) - .try_into() - .unwrap_or(u32::MAX); - let f = 2u64.saturating_pow(f).try_into().unwrap_or(i32::MAX); - let timeout = Self::BASE_DURATION - .saturating_mul(f) - .min(Self::MAX_DURATION); - - metrics::METRICS.replica_view_timeout.set_latency(timeout); - self.timeout_deadline = time::Deadline::Finite(ctx.now() + timeout); - } -} diff --git a/node/actors/bft/src/testonly/make.rs b/node/actors/bft/src/testonly/make.rs index d2b49113..13382860 100644 --- a/node/actors/bft/src/testonly/make.rs +++ b/node/actors/bft/src/testonly/make.rs @@ -1,9 +1,23 @@ //! This module contains utilities that are only meant for testing purposes. +use crate::io::InputMessage; use crate::PayloadManager; -use rand::Rng as _; +use rand::{distributions::Standard, prelude::Distribution, Rng}; use zksync_concurrency::ctx; +use zksync_concurrency::oneshot; +use zksync_consensus_network::io::ConsensusReq; use zksync_consensus_roles::validator; +// Generates a random InputMessage. +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> InputMessage { + let (send, _) = oneshot::channel(); + InputMessage::Network(ConsensusReq { + msg: rng.gen(), + ack: send, + }) + } +} + /// Produces random payload of a given size. #[derive(Debug)] pub struct RandomPayload(pub usize); diff --git a/node/actors/bft/src/testonly/mod.rs b/node/actors/bft/src/testonly/mod.rs index 504bb149..03aed7c0 100644 --- a/node/actors/bft/src/testonly/mod.rs +++ b/node/actors/bft/src/testonly/mod.rs @@ -1,33 +1,15 @@ //! This module contains utilities that are only meant for testing purposes. -use crate::io::InputMessage; -use rand::{distributions::Standard, prelude::Distribution, Rng}; -use zksync_concurrency::oneshot; -use zksync_consensus_network::io::ConsensusReq; - mod make; #[cfg(test)] mod node; #[cfg(test)] mod run; #[cfg(test)] -pub(crate) mod ut_harness; +pub mod twins; pub use make::*; #[cfg(test)] pub(crate) use node::*; #[cfg(test)] pub(crate) use run::*; -#[cfg(test)] -pub mod twins; - -// Generates a random InputMessage. -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> InputMessage { - let (send, _) = oneshot::channel(); - InputMessage::Network(ConsensusReq { - msg: rng.gen(), - ack: send, - }) - } -} diff --git a/node/actors/bft/src/testonly/run.rs b/node/actors/bft/src/testonly/run.rs index 8b06968b..f303a0d4 100644 --- a/node/actors/bft/src/testonly/run.rs +++ b/node/actors/bft/src/testonly/run.rs @@ -29,11 +29,24 @@ pub(crate) enum Network { /// Technically there are 4 phases but that results in tests timing out as /// the chance of a reaching consensus in any round goes down rapidly. /// -/// Instead we can just use two phase-partitions: one for the LeaderCommit, +/// Instead we can just use two phase-partitions: one for the LeaderProposal, /// and another for everything else. This models the typical adversarial -/// scenario of not everyone getting the QC. +/// scenario of not everyone getting the proposal. pub(crate) const NUM_PHASES: usize = 2; +/// Index of the phase in which the message appears, to decide which partitioning to apply. +fn msg_phase_number(msg: &validator::ConsensusMsg) -> usize { + use validator::ConsensusMsg; + let phase = match msg { + ConsensusMsg::LeaderProposal(_) => 0, + ConsensusMsg::ReplicaCommit(_) => 1, + ConsensusMsg::ReplicaTimeout(_) => 1, + ConsensusMsg::ReplicaNewView(_) => 1, + }; + assert!(phase < NUM_PHASES); + phase +} + /// Identify different network identities of twins by their listener port. /// They are all expected to be on localhost, but `ListenerAddr` can't be /// directly used as a map key. @@ -47,12 +60,13 @@ pub(crate) type PortSplitSchedule = Vec<[PortSplit; NUM_PHASES]>; /// Function to decide whether a message can go from a source to a target port. pub(crate) type PortRouterFn = dyn Fn(&validator::ConsensusMsg, Port, Port) -> Option + Sync; -/// A predicate to gover who can communicate to whom a given message. +/// A predicate to govern who can communicate to whom a given message. pub(crate) enum PortRouter { /// List of port splits for each view/phase, where ports in the same partition can send any message to each other. Splits(PortSplitSchedule), /// Custom routing function which can take closer control of which message can be sent in which direction, /// in order to reenact particular edge cases. + #[allow(dead_code)] Custom(Box), } @@ -264,7 +278,6 @@ async fn run_nodes_twins( // Taking these references is necessary for the `scope::run!` environment lifetime rules to compile // with `async move`, which in turn is necessary otherwise it the spawned process could not borrow `port`. // Potentially `ctx::NoCopy` could be used with `port`. - let validator_ports = &validator_ports; let sends = &sends; let stores = &stores; let gossip_targets = &gossip_targets; @@ -283,7 +296,6 @@ async fn run_nodes_twins( twins_receive_loop( ctx, router, - validator_ports, sends, TwinsGossipConfig { targets: &gossip_targets[&port], @@ -308,12 +320,11 @@ async fn run_nodes_twins( /// according to the partition schedule of the port associated with this instance. /// /// We have to simulate the gossip layer which isn't instantiated by these tests. -/// If we don't, then if a replica misses a LeaderPrepare message it won't ever get the payload +/// If we don't, then if a replica misses a LeaderProposal message it won't ever get the payload /// and won't be able to finalize the block, and won't participate further in the consensus. async fn twins_receive_loop( ctx: &ctx::Ctx, router: &PortRouter, - validator_ports: &HashMap>, sends: &HashMap>, gossip: TwinsGossipConfig<'_>, port: Port, @@ -331,7 +342,7 @@ async fn twins_receive_loop( // We need to buffer messages that cannot be delivered due to partitioning, and deliver them later. // The spec says that the network is expected to deliver messages eventually, potentially out of order, - // caveated by the fact that the actual implementation only keeps retrying the last message.. + // caveated by the fact that the actual implementation only keeps retrying the last message. // A separate issue is the definition of "later", without actually adding timing assumptions: // * If we want to allow partitions which don't have enough replicas for a quorum, and the replicas // don't move on from a view until they reach quorum, then "later" could be defined by so many @@ -341,12 +352,7 @@ async fn twins_receive_loop( // can move on to the next view, in which a new partition configuration will allow them to broadcast // to previously isolated peers. // * One idea is to wait until replica A wants to send to replica B in a view when they are no longer - // partitioned, and then unstash all previous A-to-B messages. This would _not_ work with HotStuff - // out of the box, because replicas only communicate with their leader, so if for example B missed - // a LeaderCommit from A in an earlier view, B will not respond to the LeaderPrepare from C because - // they can't commit the earlier block until they get a new message from A. However since - // https://github.com/matter-labs/era-consensus/pull/119 the ReplicaPrepare messages are broadcasted, - // so we shouldn't have to wait long for A to unstash its messages to B. + // partitioned, and then unstash all previous A-to-B messages. // * If that wouldn't be acceptable then we could have some kind of global view of stashed messages // and unstash them as soon as someone moves on to a new view. let mut stashes: HashMap> = HashMap::new(); @@ -413,24 +419,12 @@ async fn twins_receive_loop( } }; - match message.recipient { - io::Target::Broadcast => { - tracing::info!("broadcasting view={view} from={port} kind={kind}"); - for target_port in sends.keys() { - send_or_stash(can_send(*target_port)?, *target_port, msg()); - } - } - io::Target::Validator(ref v) => { - let target_ports = &validator_ports[v]; - tracing::info!( - "unicasting view={view} from={port} target={target_ports:?} kind={kind}" - ); - for target_port in target_ports { - send_or_stash(can_send(*target_port)?, *target_port, msg()); - } - } + tracing::info!("broadcasting view={view} from={port} kind={kind}"); + for target_port in sends.keys() { + send_or_stash(can_send(*target_port)?, *target_port, msg()); } } + Ok(()) } @@ -510,27 +504,20 @@ fn output_msg_label(msg: &io::OutputMessage) -> &str { fn output_msg_commit_qc(msg: &io::OutputMessage) -> Option<&validator::CommitQC> { use validator::ConsensusMsg; - match msg { + + let justification = match msg { io::OutputMessage::Consensus(cr) => match &cr.msg.msg { - ConsensusMsg::ReplicaPrepare(rp) => rp.high_qc.as_ref(), - ConsensusMsg::LeaderPrepare(lp) => lp.justification.high_qc(), - ConsensusMsg::ReplicaCommit(_) => None, - ConsensusMsg::LeaderCommit(lc) => Some(&lc.justification), + ConsensusMsg::ReplicaTimeout(msg) => return msg.high_qc.as_ref(), + ConsensusMsg::ReplicaCommit(_) => return None, + ConsensusMsg::ReplicaNewView(msg) => &msg.justification, + ConsensusMsg::LeaderProposal(msg) => &msg.justification, }, - } -} - -/// Index of the phase in which the message appears, to decide which partitioning to apply. -fn msg_phase_number(msg: &validator::ConsensusMsg) -> usize { - use validator::ConsensusMsg; - let phase = match msg { - ConsensusMsg::ReplicaPrepare(_) => 0, - ConsensusMsg::LeaderPrepare(_) => 0, - ConsensusMsg::ReplicaCommit(_) => 0, - ConsensusMsg::LeaderCommit(_) => 1, }; - assert!(phase < NUM_PHASES); - phase + + match justification { + validator::ProposalJustification::Commit(commit_qc) => Some(commit_qc), + validator::ProposalJustification::Timeout(timeout_qc) => timeout_qc.high_qc(), + } } struct TwinsGossipMessage { diff --git a/node/actors/bft/src/testonly/ut_harness.rs b/node/actors/bft/src/testonly/ut_harness.rs deleted file mode 100644 index 53b2acbe..00000000 --- a/node/actors/bft/src/testonly/ut_harness.rs +++ /dev/null @@ -1,334 +0,0 @@ -use crate::{ - io::OutputMessage, - leader, - leader::{replica_commit, replica_prepare}, - replica, - replica::{leader_commit, leader_prepare}, - testonly, Config, PayloadManager, -}; -use assert_matches::assert_matches; -use std::sync::Arc; -use zksync_concurrency::{ctx, sync::prunable_mpsc}; -use zksync_consensus_network as network; -use zksync_consensus_roles::validator; -use zksync_consensus_storage::{ - testonly::{in_memory, TestMemoryStorage}, - BlockStoreRunner, -}; -use zksync_consensus_utils::enum_util::Variant; - -pub(crate) const MAX_PAYLOAD_SIZE: usize = 1000; - -/// `UTHarness` provides various utilities for unit tests. -/// It is designed to simplify the setup and execution of test cases by encapsulating -/// common testing functionality. -/// -/// It should be instantiated once for every test case. -#[cfg(test)] -pub(crate) struct UTHarness { - pub(crate) leader: leader::StateMachine, - pub(crate) replica: replica::StateMachine, - pub(crate) keys: Vec, - pub(crate) leader_send: prunable_mpsc::Sender, - pipe: ctx::channel::UnboundedReceiver, -} - -impl UTHarness { - /// Creates a new `UTHarness` with the specified validator set size. - pub(crate) async fn new( - ctx: &ctx::Ctx, - num_validators: usize, - ) -> (UTHarness, BlockStoreRunner) { - Self::new_with_payload( - ctx, - num_validators, - Box::new(testonly::RandomPayload(MAX_PAYLOAD_SIZE)), - ) - .await - } - - pub(crate) async fn new_with_payload( - ctx: &ctx::Ctx, - num_validators: usize, - payload_manager: Box, - ) -> (UTHarness, BlockStoreRunner) { - let rng = &mut ctx.rng(); - let setup = validator::testonly::Setup::new(rng, num_validators); - let store = TestMemoryStorage::new(ctx, &setup).await; - let (send, recv) = ctx::channel::unbounded(); - - let cfg = Arc::new(Config { - secret_key: setup.validator_keys[0].clone(), - block_store: store.blocks.clone(), - replica_store: Box::new(in_memory::ReplicaStore::default()), - payload_manager, - max_payload_size: MAX_PAYLOAD_SIZE, - }); - let (leader, leader_send) = leader::StateMachine::new(ctx, cfg.clone(), send.clone()); - let (replica, _) = replica::StateMachine::start(ctx, cfg.clone(), send.clone()) - .await - .unwrap(); - let mut this = UTHarness { - leader, - replica, - pipe: recv, - keys: setup.validator_keys.clone(), - leader_send, - }; - let _: validator::Signed = this.try_recv().unwrap(); - (this, store.runner) - } - - /// Creates a new `UTHarness` with minimally-significant validator set size. - pub(crate) async fn new_many(ctx: &ctx::Ctx) -> (UTHarness, BlockStoreRunner) { - let num_validators = 6; - let (util, runner) = UTHarness::new(ctx, num_validators).await; - assert!(util.genesis().validators.max_faulty_weight() > 0); - (util, runner) - } - - /// Triggers replica timeout, validates the new validator::ReplicaPrepare - /// then executes the whole new view to make sure that the consensus - /// recovers after a timeout. - pub(crate) async fn produce_block_after_timeout(&mut self, ctx: &ctx::Ctx) { - let want = validator::ReplicaPrepare { - view: validator::View { - genesis: self.genesis().hash(), - number: self.replica.view.next(), - }, - high_qc: self.replica.high_qc.clone(), - high_vote: self.replica.high_vote.clone(), - }; - let replica_prepare = self.process_replica_timeout(ctx).await; - assert_eq!(want, replica_prepare.msg); - self.produce_block(ctx).await; - } - - /// Produces a block, by executing the full view. - pub(crate) async fn produce_block(&mut self, ctx: &ctx::Ctx) { - let msg = self.new_leader_commit(ctx).await; - self.process_leader_commit(ctx, self.sign(msg)) - .await - .unwrap(); - } - - pub(crate) fn owner_key(&self) -> &validator::SecretKey { - &self.replica.config.secret_key - } - - pub(crate) fn sign>(&self, msg: V) -> validator::Signed { - self.replica.config.secret_key.sign_msg(msg) - } - - pub(crate) fn set_owner_as_view_leader(&mut self) { - let mut view = self.replica.view; - while self.view_leader(view) != self.owner_key().public() { - view = view.next(); - } - self.replica.view = view; - } - - pub(crate) fn replica_view(&self) -> validator::View { - validator::View { - genesis: self.genesis().hash(), - number: self.replica.view, - } - } - - pub(crate) fn new_replica_prepare(&mut self) -> validator::ReplicaPrepare { - self.set_owner_as_view_leader(); - validator::ReplicaPrepare { - view: self.replica_view(), - high_vote: self.replica.high_vote.clone(), - high_qc: self.replica.high_qc.clone(), - } - } - - pub(crate) fn new_current_replica_commit(&self) -> validator::ReplicaCommit { - validator::ReplicaCommit { - view: self.replica_view(), - proposal: self.replica.high_qc.as_ref().unwrap().message.proposal, - } - } - - pub(crate) async fn new_leader_prepare(&mut self, ctx: &ctx::Ctx) -> validator::LeaderPrepare { - let msg = self.new_replica_prepare(); - self.process_replica_prepare_all(ctx, msg).await.msg - } - - pub(crate) async fn new_replica_commit(&mut self, ctx: &ctx::Ctx) -> validator::ReplicaCommit { - let msg = self.new_leader_prepare(ctx).await; - self.process_leader_prepare(ctx, self.sign(msg)) - .await - .unwrap() - .msg - } - - pub(crate) async fn new_leader_commit(&mut self, ctx: &ctx::Ctx) -> validator::LeaderCommit { - let msg = self.new_replica_commit(ctx).await; - self.process_replica_commit_all(ctx, msg).await.msg - } - - pub(crate) async fn process_leader_prepare( - &mut self, - ctx: &ctx::Ctx, - msg: validator::Signed, - ) -> Result, leader_prepare::Error> { - self.replica.process_leader_prepare(ctx, msg).await?; - Ok(self.try_recv().unwrap()) - } - - pub(crate) async fn process_leader_commit( - &mut self, - ctx: &ctx::Ctx, - msg: validator::Signed, - ) -> Result, leader_commit::Error> { - self.replica.process_leader_commit(ctx, msg).await?; - Ok(self.try_recv().unwrap()) - } - - #[allow(clippy::result_large_err)] - pub(crate) async fn process_replica_prepare( - &mut self, - ctx: &ctx::Ctx, - msg: validator::Signed, - ) -> Result>, replica_prepare::Error> { - let prepare_qc = self.leader.prepare_qc.subscribe(); - self.leader.process_replica_prepare(ctx, msg).await?; - if prepare_qc.has_changed().unwrap() { - let prepare_qc = prepare_qc.borrow().clone().unwrap(); - leader::StateMachine::propose( - ctx, - &self.leader.config, - prepare_qc, - &self.leader.outbound_pipe, - ) - .await - .unwrap(); - } - Ok(self.try_recv()) - } - - pub(crate) async fn process_replica_prepare_all( - &mut self, - ctx: &ctx::Ctx, - msg: validator::ReplicaPrepare, - ) -> validator::Signed { - let mut leader_prepare = None; - let msgs: Vec<_> = self.keys.iter().map(|k| k.sign_msg(msg.clone())).collect(); - let mut first_match = true; - for (i, msg) in msgs.into_iter().enumerate() { - let res = self.process_replica_prepare(ctx, msg).await; - match ( - (i + 1) as u64 * self.genesis().validators.iter().next().unwrap().weight - < self.genesis().validators.threshold(), - first_match, - ) { - (true, _) => assert!(res.unwrap().is_none()), - (false, true) => { - first_match = false; - leader_prepare = res.unwrap() - } - (false, false) => assert_matches!(res, Err(replica_prepare::Error::Old { .. })), - } - } - leader_prepare.unwrap() - } - - pub(crate) async fn process_replica_commit( - &mut self, - ctx: &ctx::Ctx, - msg: validator::Signed, - ) -> Result>, replica_commit::Error> { - self.leader.process_replica_commit(ctx, msg)?; - Ok(self.try_recv()) - } - - async fn process_replica_commit_all( - &mut self, - ctx: &ctx::Ctx, - msg: validator::ReplicaCommit, - ) -> validator::Signed { - let mut first_match = true; - for (i, key) in self.keys.iter().enumerate() { - let res = self - .leader - .process_replica_commit(ctx, key.sign_msg(msg.clone())); - match ( - (i + 1) as u64 * self.genesis().validators.iter().next().unwrap().weight - < self.genesis().validators.threshold(), - first_match, - ) { - (true, _) => res.unwrap(), - (false, true) => { - first_match = false; - res.unwrap() - } - (false, false) => assert_matches!(res, Err(replica_commit::Error::Old { .. })), - } - } - self.try_recv().unwrap() - } - - fn try_recv>(&mut self) -> Option> { - self.pipe.try_recv().map(|message| match message { - OutputMessage::Network(network::io::ConsensusInputMessage { message, .. }) => { - message.cast().unwrap() - } - }) - } - - pub(crate) async fn process_replica_timeout( - &mut self, - ctx: &ctx::Ctx, - ) -> validator::Signed { - self.replica.start_new_view(ctx).await.unwrap(); - self.try_recv().unwrap() - } - - pub(crate) fn leader_phase(&self) -> validator::Phase { - self.leader.phase - } - - pub(crate) fn view_leader(&self, view: validator::ViewNumber) -> validator::PublicKey { - self.genesis().view_leader(view) - } - - pub(crate) fn genesis(&self) -> &validator::Genesis { - self.replica.config.genesis() - } - - pub(crate) fn new_commit_qc( - &self, - mutate_fn: impl FnOnce(&mut validator::ReplicaCommit), - ) -> validator::CommitQC { - let mut msg = self.new_current_replica_commit(); - mutate_fn(&mut msg); - let mut qc = validator::CommitQC::new(msg, self.genesis()); - for key in &self.keys { - qc.add(&key.sign_msg(qc.message.clone()), self.genesis()) - .unwrap(); - } - qc - } - - pub(crate) fn new_prepare_qc( - &mut self, - mutate_fn: impl FnOnce(&mut validator::ReplicaPrepare), - ) -> validator::PrepareQC { - let mut msg = self.new_replica_prepare(); - mutate_fn(&mut msg); - let mut qc = validator::PrepareQC::new(msg.view.clone()); - for key in &self.keys { - qc.add(&key.sign_msg(msg.clone()), self.genesis()).unwrap(); - } - qc - } - - pub(crate) fn leader_send(&self, msg: validator::Signed) { - self.leader_send.send(network::io::ConsensusReq { - msg, - ack: zksync_concurrency::oneshot::channel().0, - }); - } -} diff --git a/node/actors/bft/src/tests.rs b/node/actors/bft/src/tests.rs deleted file mode 100644 index 13e0dd1a..00000000 --- a/node/actors/bft/src/tests.rs +++ /dev/null @@ -1,610 +0,0 @@ -use crate::testonly::{ - twins::{Cluster, HasKey, ScenarioGenerator, Twin}, - ut_harness::UTHarness, - Behavior, Network, Port, PortRouter, PortSplitSchedule, Test, TestError, NUM_PHASES, -}; -use assert_matches::assert_matches; -use std::collections::HashMap; -use test_casing::{cases, test_casing, TestCases}; -use zksync_concurrency::{ctx, scope, time}; -use zksync_consensus_network::testonly::new_configs_for_validators; -use zksync_consensus_roles::validator::{ - self, - testonly::{Setup, SetupSpec}, - LeaderSelectionMode, PublicKey, SecretKey, ViewNumber, -}; - -async fn run_test(behavior: Behavior, network: Network) { - tokio::time::pause(); - let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(30)); - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - - const NODES: usize = 11; - let mut nodes = vec![(behavior, 1u64); NODES]; - // validator::threshold(NODES) will calculate required nodes to validate a message - // given each node weight is 1 - let honest_nodes_amount = validator::threshold(NODES as u64) as usize; - for n in &mut nodes[0..honest_nodes_amount] { - n.0 = Behavior::Honest; - } - Test { - network, - nodes, - blocks_to_finalize: 10, - } - .run(ctx) - .await - .unwrap() -} - -#[tokio::test] -async fn honest_real_network() { - run_test(Behavior::Honest, Network::Real).await -} - -#[tokio::test] -async fn offline_real_network() { - run_test(Behavior::Offline, Network::Real).await -} - -/// Testing liveness after the network becomes idle with leader having no cached prepare messages for the current view. -#[tokio::test] -async fn timeout_leader_no_prepares() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - util.new_replica_prepare(); - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Testing liveness after the network becomes idle with leader having some cached prepare messages for the current view. -#[tokio::test] -async fn timeout_leader_some_prepares() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - let replica_prepare = util.new_replica_prepare(); - assert!(util - .process_replica_prepare(ctx, util.sign(replica_prepare)) - .await - .unwrap() - .is_none()); - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Testing liveness after the network becomes idle with leader in commit phase. -#[tokio::test] -async fn timeout_leader_in_commit() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.new_leader_prepare(ctx).await; - // Leader is in `Phase::Commit`, but should still accept prepares from newer views. - assert_eq!(util.leader.phase, validator::Phase::Commit); - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Testing liveness after the network becomes idle with replica in commit phase. -#[tokio::test] -async fn timeout_replica_in_commit() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.new_replica_commit(ctx).await; - // Leader is in `Phase::Commit`, but should still accept prepares from newer views. - assert_eq!(util.leader.phase, validator::Phase::Commit); - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Testing liveness after the network becomes idle with leader having some cached commit messages for the current view. -#[tokio::test] -async fn timeout_leader_some_commits() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - let replica_commit = util.new_replica_commit(ctx).await; - assert!(util - .process_replica_commit(ctx, util.sign(replica_commit)) - .await - .unwrap() - .is_none()); - // Leader is in `Phase::Commit`, but should still accept prepares from newer views. - assert_eq!(util.leader_phase(), validator::Phase::Commit); - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Testing liveness after the network becomes idle with leader in a consecutive prepare phase. -#[tokio::test] -async fn timeout_leader_in_consecutive_prepare() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::RealClock); - scope::run!(ctx, |ctx, s| async { - let (mut util, runner) = UTHarness::new_many(ctx).await; - s.spawn_bg(runner.run(ctx)); - - util.new_leader_commit(ctx).await; - util.produce_block_after_timeout(ctx).await; - Ok(()) - }) - .await - .unwrap(); -} - -/// Not being able to propose a block shouldn't cause a deadlock. -#[tokio::test] -async fn non_proposing_leader() { - zksync_concurrency::testonly::abort_on_panic(); - let ctx = &ctx::test_root(&ctx::AffineClock::new(5.)); - Test { - network: Network::Real, - nodes: vec![(Behavior::Honest, 1), (Behavior::HonestNotProposing, 1)], - blocks_to_finalize: 10, - } - .run(ctx) - .await - .unwrap() -} - -/// Run Twins scenarios without actual twins, and with so few nodes that all -/// of them are required for a quorum, which means (currently) there won't be -/// any partitions. -/// -/// This should be a simple sanity check that the network works and consensus -/// is achieved under the most favourable conditions. -#[test_casing(10,0..10)] -#[tokio::test] -async fn twins_network_wo_twins_wo_partitions(num_reseeds: usize) { - tokio::time::pause(); - // n<6 implies f=0 and q=n - run_twins(5, 0, TwinsScenarios::Reseeds(num_reseeds)) - .await - .unwrap(); -} - -/// Run Twins scenarios without actual twins, but enough replicas that partitions -/// can play a role, isolating certain nodes (potentially the leader) in some -/// rounds. -/// -/// This should be a sanity check that without Byzantine behaviour the consensus -/// is resilient to temporary network partitions. -#[test_casing(5,0..5)] -#[tokio::test] -async fn twins_network_wo_twins_w_partitions(num_reseeds: usize) { - tokio::time::pause(); - // n=6 implies f=1 and q=5; 6 is the minimum where partitions are possible. - run_twins(6, 0, TwinsScenarios::Reseeds(num_reseeds)) - .await - .unwrap(); -} - -/// Test cases with 1 twin, with 6-10 replicas, 10 scenarios each. -const CASES_TWINS_1: TestCases<(usize, usize)> = cases! { - (6..=10).flat_map(|num_replicas| (0..10).map(move |num_reseeds| (num_replicas, num_reseeds))) -}; - -/// Run Twins scenarios with random number of nodes and 1 twin. -#[test_casing(50, CASES_TWINS_1)] -#[tokio::test] -async fn twins_network_w1_twins_w_partitions(num_replicas: usize, num_reseeds: usize) { - tokio::time::pause(); - // n>=6 implies f>=1 and q=n-f - // let num_honest = validator::threshold(num_replicas as u64) as usize; - // let max_faulty = num_replicas - num_honest; - // let num_twins = rng.gen_range(1..=max_faulty); - run_twins(num_replicas, 1, TwinsScenarios::Reseeds(num_reseeds)) - .await - .unwrap(); -} - -/// Run Twins scenarios with higher number of nodes and 2 twins. -#[test_casing(5,0..5)] -#[tokio::test] -async fn twins_network_w2_twins_w_partitions(num_reseeds: usize) { - tokio::time::pause(); - // n>=11 implies f>=2 and q=n-f - run_twins(11, 2, TwinsScenarios::Reseeds(num_reseeds)) - .await - .unwrap(); -} - -/// Run Twins scenario with more twins than tolerable and expect it to fail. -#[tokio::test] -async fn twins_network_to_fail() { - tokio::time::pause(); - // With n=5 f=0, so 1 twin means more faulty nodes than expected. - assert_matches!( - run_twins(5, 1, TwinsScenarios::Multiple(100)).await, - Err(TestError::BlockConflict) - ); -} - -/// Govern how many scenarios to execute in the test. -enum TwinsScenarios { - /// Execute N scenarios in a loop. - /// - /// Use this when looking for a counter example, ie. a scenario where consensus fails. - Multiple(usize), - /// Execute 1 scenario after doing N reseeds of the RNG. - /// - /// Use this with the `#[test_casing]` macro to turn scenarios into separate test cases. - Reseeds(usize), -} - -/// Create network configuration for a given number of replicas and twins and run [Test], -async fn run_twins( - num_replicas: usize, - num_twins: usize, - scenarios: TwinsScenarios, -) -> Result<(), TestError> { - zksync_concurrency::testonly::abort_on_panic(); - - // A single scenario with 11 replicas took 3-5 seconds. - // Panic on timeout; works with `cargo nextest` and the `abort_on_panic` above. - let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(60)); - let ctx = &ctx::test_root(&ctx::RealClock); - - #[derive(PartialEq, Debug)] - struct Replica { - id: i64, // non-zero ID - public_key: PublicKey, - secret_key: SecretKey, - } - - impl HasKey for Replica { - type Key = PublicKey; - - fn key(&self) -> &Self::Key { - &self.public_key - } - } - - impl Twin for Replica { - fn to_twin(&self) -> Self { - Self { - id: -self.id, - public_key: self.public_key.clone(), - secret_key: self.secret_key.clone(), - } - } - } - - let (num_scenarios, num_reseeds) = match scenarios { - TwinsScenarios::Multiple(n) => (n, 0), - TwinsScenarios::Reseeds(n) => (1, n), - }; - - // Keep scenarios separate by generating a different RNG many times. - let mut rng = ctx.rng(); - for _ in 0..num_reseeds { - rng = ctx.rng(); - } - let rng = &mut rng; - - // The existing test machinery uses the number of finalized blocks as an exit criteria. - let blocks_to_finalize = 3; - // The test is going to disrupt the communication by partitioning nodes, - // where the leader might not be in a partition with enough replicas to - // form a quorum, therefore to allow N blocks to be finalized we need to - // go longer. - let num_rounds = blocks_to_finalize * 10; - // The paper considers 2 or 3 partitions enough. - let max_partitions = 3; - - // Every validator has equal power of 1. - const WEIGHT: u64 = 1; - let mut spec = SetupSpec::new_with_weights(rng, vec![WEIGHT; num_replicas]); - - let replicas = spec - .validator_weights - .iter() - .enumerate() - .map(|(i, (sk, _))| Replica { - id: i as i64 + 1, - public_key: sk.public(), - secret_key: sk.clone(), - }) - .collect::>(); - - let cluster = Cluster::new(replicas, num_twins); - let scenarios = ScenarioGenerator::<_, NUM_PHASES>::new(&cluster, num_rounds, max_partitions); - - // Gossip with more nodes than what can be faulty. - let gossip_peers = num_twins + 1; - - // Create network config for all nodes in the cluster (assigns unique network addresses). - let nets = new_configs_for_validators( - rng, - cluster.nodes().iter().map(|r| &r.secret_key), - gossip_peers, - ); - - let node_to_port = cluster - .nodes() - .iter() - .zip(nets.iter()) - .map(|(node, net)| (node.id, net.server_addr.port())) - .collect::>(); - - assert_eq!(node_to_port.len(), cluster.num_nodes()); - - // Every network needs a behaviour. They are all honest, just some might be duplicated. - let nodes = vec![(Behavior::Honest, WEIGHT); cluster.num_nodes()]; - - // Reuse the same cluster and network setup to run a few scenarios. - for i in 0..num_scenarios { - // Generate a permutation of partitions and leaders for the given number of rounds. - let scenario = scenarios.generate_one(rng); - - // Assign the leadership schedule to the consensus. - spec.leader_selection = - LeaderSelectionMode::Rota(scenario.rounds.iter().map(|rc| rc.leader.clone()).collect()); - - // Generate a new setup with this leadership schedule. - let setup = Setup::from_spec(rng, spec.clone()); - - // Create a network with the partition schedule of the scenario. - let splits: PortSplitSchedule = scenario - .rounds - .iter() - .map(|rc| { - std::array::from_fn(|i| { - rc.phase_partitions[i] - .iter() - .map(|p| p.iter().map(|r| node_to_port[&r.id]).collect()) - .collect() - }) - }) - .collect(); - - tracing::info!( - "num_replicas={num_replicas} num_twins={num_twins} num_nodes={} scenario={i}", - cluster.num_nodes() - ); - - // Debug output of round schedule. - for (r, rc) in scenario.rounds.iter().enumerate() { - // Let's just consider the partition of the LeaderCommit phase, for brevity's sake. - let partitions = &splits[r].last().unwrap(); - - let leader_ports = cluster - .nodes() - .iter() - .filter(|n| n.public_key == *rc.leader) - .map(|n| node_to_port[&n.id]) - .collect::>(); - - let leader_partition_sizes = leader_ports - .iter() - .map(|lp| partitions.iter().find(|p| p.contains(lp)).unwrap().len()) - .collect::>(); - - let leader_isolated = leader_partition_sizes - .iter() - .all(|s| *s < cluster.quorum_size()); - - tracing::info!("round={r} partitions={partitions:?} leaders={leader_ports:?} leader_partition_sizes={leader_partition_sizes:?} leader_isolated={leader_isolated}"); - } - - Test { - network: Network::Twins(PortRouter::Splits(splits)), - nodes: nodes.clone(), - blocks_to_finalize, - } - .run_with_config(ctx, nets.clone(), &setup) - .await? - } - - Ok(()) -} - -/// Test a liveness issue where some validators have the HighQC but don't have the block payload and have to wait for it, -/// while some other validators have the payload but don't have the HighQC and cannot finalize the block, and therefore -/// don't gossip it, which causes a deadlock unless the one with the HighQC moves on and broadcasts what they have, which -/// should cause the others to finalize the block and gossip the payload to them in turn. -#[tokio::test] -async fn test_wait_for_finalized_deadlock() { - // These are the conditions for the deadlock to occur: - // * The problem happens in the handling of LeaderPrepare where the replica waits for the previous block in the justification. - // * For that the replica needs to receive a proposal from a leader that knows the previous block is finalized. - // * For that the leader needs to receive a finalized proposal from an earlier leader, but this proposal did not make it to the replica. - // * Both leaders need to die and never communicate the HighQC they know about to anybody else. - // * The replica has the HighQC but not the payload, and all other replicas might have the payload, but not the HighQC. - // * With two leaders down, and the replica deadlocked, we must lose quorum, so the other nodes cannot repropose the missing block either. - // * In order for 2 leaders to be dow and quorum still be possible, we need at least 11 nodes. - - // Here are a series of steps to reproduce the issue: - // 1. Say we have 11 nodes: [0,1,2,3,4,5,6,7,8,9,10], taking turns leading the views in that order; we need 9 nodes for quorum. The first view is view 1 lead by node 1. - // 2. Node 1 sends LeaderPropose with block 1 to nodes [1-9] and puts together a HighQC. - // 3. Node 1 sends the LeaderCommit to node 2, then dies. - // 4. Node 2 sends LeaderPropose with block 2 to nodes [0, 10], then dies. - // 5. Nodes [0, 10] get stuck processing LeaderPropose because they are waiting for block 1 to appear in their stores. - // 6. Node 3 cannot gather 9 ReplicaPrepare messages for a quorum because nodes [1,2] are down and [0,10] are blocking. Consensus stalls. - - // To simulate this with the Twins network we need to use a custom routing function, because the 2nd leader mustn't broadcast the HighQC - // to its peers, but it must receive their ReplicaPrepare's to be able to construct the PrepareQC; because of this the simple split schedule - // would not be enough as it allows sending messages in both directions. - - // We need 11 nodes so we can turn 2 leaders off. - let num_replicas = 11; - // Let's wait for the first two blocks to be finalised. - // Although theoretically node 1 will be dead after view 1, it will still receive messages and gossip. - let blocks_to_finalize = 2; - // We need more than 1 gossip peer, otherwise the chain of gossip triggers in the Twins network won't kick in, - // and while node 0 will gossip to node 1, node 1 will not send it to node 2, and the test will fail. - let gossip_peers = 2; - - run_with_custom_router( - num_replicas, - gossip_peers, - blocks_to_finalize, - |port_to_id| { - PortRouter::Custom(Box::new(move |msg, from, to| { - use validator::ConsensusMsg::*; - // Map ports back to logical node ID - let from = port_to_id[&from]; - let to = port_to_id[&to]; - let view_number = msg.view().number; - - // If we haven't finalised the blocks in the first few rounds, we failed. - if view_number.0 > 7 { - return None; - } - - // Sending to self is ok. - // If this wasn't here the test would pass even without adding a timeout in process_leader_prepare. - // The reason is that node 2 would move to view 2 as soon as it finalises block 1, but then timeout - // and move to view 3 before they receive any of the ReplicaPrepare from the others, who are still - // waiting to timeout in view 1. By sending ReplicaPrepare to itself it seems to wait or propose. - // Maybe the HighQC doesn't make it from its replica::StateMachine into its leader::StateMachine otherwise. - if from == to { - return Some(true); - } - - let can_send = match view_number { - ViewNumber(1) => { - match from { - // Current leader - 1 => match msg { - // Send the proposal to a subset of nodes - LeaderPrepare(_) => to != 0 && to != 10, - // Send the commit to the next leader only - LeaderCommit(_) => to == 2, - _ => true, - }, - // Replicas - _ => true, - } - } - ViewNumber(2) => match from { - // Previous leader is dead - 1 => false, - // Current leader - 2 => match msg { - // Don't send out the HighQC to the others - ReplicaPrepare(_) => false, - // Send the proposal to the ones which didn't get the previous one - LeaderPrepare(_) => to == 0 || to == 10, - _ => true, - }, - // Replicas - _ => true, - }, - // Previous leaders dead - _ => from != 1 && from != 2, - }; - - // eprintln!( - // "view={view_number} from={from} to={to} kind={} can_send={can_send}", - // msg.label() - // ); - - Some(can_send) - })) - }, - ) - .await - .unwrap(); -} - -/// Run a test with the Twins network controlling exactly who can send to whom in each round. -/// -/// The input for the router is a mapping from port to the index of nodes starting from 0. -/// The first view to be executed is view 1 and will have the node 1 as its leader, and so on, -/// so a routing function can expect view `i` to be lead by node `i`, and express routing -/// rules with the logic IDs. -async fn run_with_custom_router( - num_replicas: usize, - gossip_peers: usize, - blocks_to_finalize: usize, - make_router: impl FnOnce(HashMap) -> PortRouter, -) -> Result<(), TestError> { - tokio::time::pause(); - zksync_concurrency::testonly::abort_on_panic(); - let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(60)); - let ctx = &ctx::test_root(&ctx::RealClock); - - let rng = &mut ctx.rng(); - - let mut spec = SetupSpec::new(rng, num_replicas); - - let nodes = spec - .validator_weights - .iter() - .map(|(_, w)| (Behavior::Honest, *w)) - .collect(); - - let nets = new_configs_for_validators( - rng, - spec.validator_weights.iter().map(|(sk, _)| sk), - gossip_peers, - ); - - // Assign the validator rota to be in the order of appearance, not ordered by public key. - spec.leader_selection = LeaderSelectionMode::Rota( - spec.validator_weights - .iter() - .map(|(sk, _)| sk.public()) - .collect(), - ); - - let setup = Setup::from_spec(rng, spec); - - let port_to_id = nets - .iter() - .enumerate() - .map(|(i, net)| (net.server_addr.port(), i)) - .collect::>(); - - // Sanity check the leader schedule - { - let pk = setup.genesis.view_leader(ViewNumber(1)); - let cfg = nets - .iter() - .find(|net| net.validator_key.as_ref().unwrap().public() == pk) - .unwrap(); - let port = cfg.server_addr.port(); - assert_eq!(port_to_id[&port], 1); - } - - Test { - network: Network::Twins(make_router(port_to_id)), - nodes, - blocks_to_finalize, - } - .run_with_config(ctx, nets, &setup) - .await -} diff --git a/node/actors/bft/src/tests/mod.rs b/node/actors/bft/src/tests/mod.rs new file mode 100644 index 00000000..9676665f --- /dev/null +++ b/node/actors/bft/src/tests/mod.rs @@ -0,0 +1,53 @@ +use crate::testonly::{Behavior, Network, Test}; +use zksync_concurrency::{ctx, time}; +use zksync_consensus_roles::validator; + +mod twins; + +async fn run_test(behavior: Behavior, network: Network) { + tokio::time::pause(); + let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(60)); + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::RealClock); + + const NODES: usize = 11; + let mut nodes = vec![(behavior, 1u64); NODES]; + // validator::threshold(NODES) will calculate required nodes to validate a message + // given each node weight is 1 + let honest_nodes_amount = validator::quorum_threshold(NODES as u64) as usize; + for n in &mut nodes[0..honest_nodes_amount] { + n.0 = Behavior::Honest; + } + Test { + network, + nodes, + blocks_to_finalize: 10, + } + .run(ctx) + .await + .unwrap() +} + +#[tokio::test] +async fn honest_real_network() { + run_test(Behavior::Honest, Network::Real).await +} + +#[tokio::test] +async fn offline_real_network() { + run_test(Behavior::Offline, Network::Real).await +} + +#[tokio::test] +async fn honest_not_proposing_real_network() { + zksync_concurrency::testonly::abort_on_panic(); + let ctx = &ctx::test_root(&ctx::AffineClock::new(5.)); + Test { + network: Network::Real, + nodes: vec![(Behavior::Honest, 1), (Behavior::HonestNotProposing, 1)], + blocks_to_finalize: 10, + } + .run(ctx) + .await + .unwrap() +} diff --git a/node/actors/bft/src/tests/twins.rs b/node/actors/bft/src/tests/twins.rs new file mode 100644 index 00000000..45603e51 --- /dev/null +++ b/node/actors/bft/src/tests/twins.rs @@ -0,0 +1,265 @@ +use crate::testonly::{ + twins::{Cluster, HasKey, ScenarioGenerator, Twin}, + Behavior, Network, PortRouter, PortSplitSchedule, Test, TestError, NUM_PHASES, +}; +use assert_matches::assert_matches; +use std::collections::HashMap; +use test_casing::{cases, test_casing, TestCases}; +use zksync_concurrency::{ctx, time}; +use zksync_consensus_network::testonly::new_configs_for_validators; +use zksync_consensus_roles::validator::{ + testonly::{Setup, SetupSpec}, + LeaderSelectionMode, PublicKey, SecretKey, +}; + +/// Govern how many scenarios to execute in the test. +enum TwinsScenarios { + /// Execute N scenarios in a loop. + /// + /// Use this when looking for a counter example, ie. a scenario where consensus fails. + Multiple(usize), + /// Execute 1 scenario after doing N reseeds of the RNG. + /// + /// Use this with the `#[test_casing]` macro to turn scenarios into separate test cases. + Reseeds(usize), +} + +/// Create network configuration for a given number of replicas and twins and run [Test], +async fn run_twins( + num_replicas: usize, + num_twins: usize, + scenarios: TwinsScenarios, +) -> Result<(), TestError> { + zksync_concurrency::testonly::abort_on_panic(); + + // A single scenario with 11 replicas took 3-5 seconds. + // Panic on timeout; works with `cargo nextest` and the `abort_on_panic` above. + let _guard = zksync_concurrency::testonly::set_timeout(time::Duration::seconds(90)); + let ctx = &ctx::test_root(&ctx::RealClock); + + #[derive(PartialEq, Debug)] + struct Replica { + id: i64, // non-zero ID + public_key: PublicKey, + secret_key: SecretKey, + } + + impl HasKey for Replica { + type Key = PublicKey; + + fn key(&self) -> &Self::Key { + &self.public_key + } + } + + impl Twin for Replica { + fn to_twin(&self) -> Self { + Self { + id: -self.id, + public_key: self.public_key.clone(), + secret_key: self.secret_key.clone(), + } + } + } + + let (num_scenarios, num_reseeds) = match scenarios { + TwinsScenarios::Multiple(n) => (n, 0), + TwinsScenarios::Reseeds(n) => (1, n), + }; + + // Keep scenarios separate by generating a different RNG many times. + let mut rng = ctx.rng(); + for _ in 0..num_reseeds { + rng = ctx.rng(); + } + let rng = &mut rng; + + // The existing test machinery uses the number of finalized blocks as an exit criteria. + let blocks_to_finalize = 3; + // The test is going to disrupt the communication by partitioning nodes, + // where the leader might not be in a partition with enough replicas to + // form a quorum, therefore to allow N blocks to be finalized we need to + // go longer. + let num_rounds = blocks_to_finalize * 10; + // The paper considers 2 or 3 partitions enough. + let max_partitions = 3; + + // Every validator has equal power of 1. + const WEIGHT: u64 = 1; + let mut spec = SetupSpec::new_with_weights(rng, vec![WEIGHT; num_replicas]); + + let replicas = spec + .validator_weights + .iter() + .enumerate() + .map(|(i, (sk, _))| Replica { + id: i as i64 + 1, + public_key: sk.public(), + secret_key: sk.clone(), + }) + .collect::>(); + + let cluster = Cluster::new(replicas, num_twins); + let scenarios = ScenarioGenerator::<_, NUM_PHASES>::new(&cluster, num_rounds, max_partitions); + + // Gossip with more nodes than what can be faulty. + let gossip_peers = num_twins + 1; + + // Create network config for all nodes in the cluster (assigns unique network addresses). + let nets = new_configs_for_validators( + rng, + cluster.nodes().iter().map(|r| &r.secret_key), + gossip_peers, + ); + + let node_to_port = cluster + .nodes() + .iter() + .zip(nets.iter()) + .map(|(node, net)| (node.id, net.server_addr.port())) + .collect::>(); + + assert_eq!(node_to_port.len(), cluster.num_nodes()); + + // Every network needs a behaviour. They are all honest, just some might be duplicated. + let nodes = vec![(Behavior::Honest, WEIGHT); cluster.num_nodes()]; + + // Reuse the same cluster and network setup to run a few scenarios. + for i in 0..num_scenarios { + // Generate a permutation of partitions and leaders for the given number of rounds. + let scenario = scenarios.generate_one(rng); + + // Assign the leadership schedule to the consensus. + spec.leader_selection = + LeaderSelectionMode::Rota(scenario.rounds.iter().map(|rc| rc.leader.clone()).collect()); + + // Generate a new setup with this leadership schedule. + let setup = Setup::from_spec(rng, spec.clone()); + + // Create a network with the partition schedule of the scenario. + let splits: PortSplitSchedule = scenario + .rounds + .iter() + .map(|rc| { + std::array::from_fn(|i| { + rc.phase_partitions[i] + .iter() + .map(|p| p.iter().map(|r| node_to_port[&r.id]).collect()) + .collect() + }) + }) + .collect(); + + tracing::info!( + "num_replicas={num_replicas} num_twins={num_twins} num_nodes={} scenario={i}", + cluster.num_nodes() + ); + + // Debug output of round schedule. + for (r, rc) in scenario.rounds.iter().enumerate() { + // Let's just consider the partition of the LeaderCommit phase, for brevity's sake. + let partitions = &splits[r].last().unwrap(); + + let leader_ports = cluster + .nodes() + .iter() + .filter(|n| n.public_key == *rc.leader) + .map(|n| node_to_port[&n.id]) + .collect::>(); + + let leader_partition_sizes = leader_ports + .iter() + .map(|lp| partitions.iter().find(|p| p.contains(lp)).unwrap().len()) + .collect::>(); + + let leader_isolated = leader_partition_sizes + .iter() + .all(|s| *s < cluster.quorum_size()); + + tracing::info!("round={r} partitions={partitions:?} leaders={leader_ports:?} leader_partition_sizes={leader_partition_sizes:?} leader_isolated={leader_isolated}"); + } + + Test { + network: Network::Twins(PortRouter::Splits(splits)), + nodes: nodes.clone(), + blocks_to_finalize, + } + .run_with_config(ctx, nets.clone(), &setup) + .await? + } + + Ok(()) +} + +/// Run Twins scenarios without actual twins, and with so few nodes that all +/// of them are required for a quorum, which means (currently) there won't be +/// any partitions. +/// +/// This should be a simple sanity check that the network works and consensus +/// is achieved under the most favourable conditions. +#[test_casing(10,0..10)] +#[tokio::test] +async fn twins_network_wo_twins_wo_partitions(num_reseeds: usize) { + tokio::time::pause(); + // n<6 implies f=0 and q=n + run_twins(5, 0, TwinsScenarios::Reseeds(num_reseeds)) + .await + .unwrap(); +} + +/// Run Twins scenarios without actual twins, but enough replicas that partitions +/// can play a role, isolating certain nodes (potentially the leader) in some +/// rounds. +/// +/// This should be a sanity check that without Byzantine behaviour the consensus +/// is resilient to temporary network partitions. +#[test_casing(5,0..5)] +#[tokio::test] +async fn twins_network_wo_twins_w_partitions(num_reseeds: usize) { + tokio::time::pause(); + // n=6 implies f=1 and q=5; 6 is the minimum where partitions are possible. + run_twins(6, 0, TwinsScenarios::Reseeds(num_reseeds)) + .await + .unwrap(); +} + +/// Test cases with 1 twin, with 6-10 replicas, 10 scenarios each. +const CASES_TWINS_1: TestCases<(usize, usize)> = cases! { + (6..=10).flat_map(|num_replicas| (0..10).map(move |num_reseeds| (num_replicas, num_reseeds))) +}; + +/// Run Twins scenarios with random number of nodes and 1 twin. +#[test_casing(50, CASES_TWINS_1)] +#[tokio::test] +async fn twins_network_w1_twins_w_partitions(num_replicas: usize, num_reseeds: usize) { + tokio::time::pause(); + // n>=6 implies f>=1 and q=n-f + // let num_honest = validator::threshold(num_replicas as u64) as usize; + // let max_faulty = num_replicas - num_honest; + // let num_twins = rng.gen_range(1..=max_faulty); + run_twins(num_replicas, 1, TwinsScenarios::Reseeds(num_reseeds)) + .await + .unwrap(); +} + +/// Run Twins scenarios with higher number of nodes and 2 twins. +#[test_casing(5,0..5)] +#[tokio::test] +async fn twins_network_w2_twins_w_partitions(num_reseeds: usize) { + tokio::time::pause(); + // n>=11 implies f>=2 and q=n-f + run_twins(11, 2, TwinsScenarios::Reseeds(num_reseeds)) + .await + .unwrap(); +} + +/// Run Twins scenario with more twins than tolerable and expect it to fail. +#[tokio::test] +async fn twins_network_to_fail() { + tokio::time::pause(); + assert_matches!( + // All twins! To find a conflict quicker. + run_twins(6, 6, TwinsScenarios::Multiple(150)).await, + Err(TestError::BlockConflict) + ); +} diff --git a/node/actors/network/src/consensus/mod.rs b/node/actors/network/src/consensus/mod.rs index b63a3248..7b6e3f9a 100644 --- a/node/actors/network/src/consensus/mod.rs +++ b/node/actors/network/src/consensus/mod.rs @@ -59,10 +59,10 @@ impl MsgPool { // an implementation detail of the bft crate. Consider moving // this logic there. match (&v.message.msg, &msg.message.msg) { - (M::ReplicaPrepare(_), M::ReplicaPrepare(_)) => {} + (M::LeaderProposal(_), M::LeaderProposal(_)) => {} (M::ReplicaCommit(_), M::ReplicaCommit(_)) => {} - (M::LeaderPrepare(_), M::LeaderPrepare(_)) => {} - (M::LeaderCommit(_), M::LeaderCommit(_)) => {} + (M::ReplicaNewView(_), M::ReplicaNewView(_)) => {} + (M::ReplicaTimeout(_), M::ReplicaTimeout(_)) => {} _ => return true, } // If pool contains a message of the same type which is newer, @@ -229,15 +229,8 @@ impl Network { let mut sub = self.msg_pool.subscribe(); loop { let call = consensus_cli.reserve(ctx).await?; - let msg = loop { - let msg = sub.recv(ctx).await?; - match &msg.recipient { - io::Target::Broadcast => {} - io::Target::Validator(recipient) if recipient == peer => {} - _ => continue, - } - break msg.message.clone(); - }; + let msg = sub.recv(ctx).await?.message.clone(); + s.spawn(async { let req = rpc::consensus::Req(msg); let res = call.call(ctx, &req, RESP_MAX_SIZE).await; diff --git a/node/actors/network/src/consensus/tests.rs b/node/actors/network/src/consensus/tests.rs index dc63d4ee..a34062f9 100644 --- a/node/actors/network/src/consensus/tests.rs +++ b/node/actors/network/src/consensus/tests.rs @@ -26,10 +26,10 @@ async fn test_msg_pool() { // We keep them sorted by type and view, so that it is easy to // compute the expected state of the pool after insertions. let msgs = [ - gen(&mut || M::ReplicaPrepare(rng.gen())), + gen(&mut || M::LeaderProposal(rng.gen())), gen(&mut || M::ReplicaCommit(rng.gen())), - gen(&mut || M::LeaderPrepare(rng.gen())), - gen(&mut || M::LeaderCommit(rng.gen())), + gen(&mut || M::ReplicaNewView(rng.gen())), + gen(&mut || M::ReplicaTimeout(rng.gen())), ]; // Insert messages at random. @@ -42,7 +42,6 @@ async fn test_msg_pool() { want[i] = Some(want[i].unwrap_or(0).max(j)); pool.send(Arc::new(io::ConsensusInputMessage { message: msgs[i][j].clone(), - recipient: io::Target::Broadcast, })); // Here we compare the internal state of the pool to the expected state. // Note that we compare sets of crypto hashes of messages, because the messages themselves do not @@ -310,9 +309,6 @@ async fn test_transmission() { let want: validator::Signed = want.cast().unwrap(); let in_message = io::ConsensusInputMessage { message: want.clone(), - recipient: io::Target::Validator( - nodes[1].cfg().validator_key.as_ref().unwrap().public(), - ), }; nodes[0].pipe.send(in_message.into()); @@ -355,7 +351,6 @@ async fn test_retransmission() { node0.pipe.send( io::ConsensusInputMessage { message: want.clone(), - recipient: io::Target::Broadcast, } .into(), ); diff --git a/node/actors/network/src/io.rs b/node/actors/network/src/io.rs index 9a7412f9..6166deef 100644 --- a/node/actors/network/src/io.rs +++ b/node/actors/network/src/io.rs @@ -13,7 +13,6 @@ pub enum InputMessage { #[derive(Debug, PartialEq)] pub struct ConsensusInputMessage { pub message: validator::Signed, - pub recipient: Target, } impl From for InputMessage { @@ -39,9 +38,3 @@ pub enum OutputMessage { /// Message to the Consensus actor. Consensus(ConsensusReq), } - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Target { - Validator(validator::PublicKey), - Broadcast, -} diff --git a/node/actors/network/src/testonly.rs b/node/actors/network/src/testonly.rs index 64f870d9..e6d5ca3d 100644 --- a/node/actors/network/src/testonly.rs +++ b/node/actors/network/src/testonly.rs @@ -1,9 +1,8 @@ //! Testonly utilities. #![allow(dead_code)] use crate::{ - gossip::attestation, - io::{ConsensusInputMessage, Target}, - Config, GossipConfig, Network, RpcConfig, Runner, + gossip::attestation, io::ConsensusInputMessage, Config, GossipConfig, Network, RpcConfig, + Runner, }; use rand::{ distributions::{Distribution, Standard}, @@ -21,21 +20,9 @@ use zksync_consensus_roles::{node, validator}; use zksync_consensus_storage::BlockStore; use zksync_consensus_utils::pipe; -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> Target { - match rng.gen_range(0..2) { - 0 => Target::Broadcast, - _ => Target::Validator(rng.gen()), - } - } -} - impl Distribution for Standard { fn sample(&self, rng: &mut R) -> ConsensusInputMessage { - ConsensusInputMessage { - message: rng.gen(), - recipient: rng.gen(), - } + ConsensusInputMessage { message: rng.gen() } } } diff --git a/node/deny.toml b/node/deny.toml index b5bda861..b4c9a367 100644 --- a/node/deny.toml +++ b/node/deny.toml @@ -18,6 +18,7 @@ allow = [ "MIT", "OpenSSL", "Unicode-DFS-2016", + "Unicode-3.0", # Weak copyleft licenses "MPL-2.0", ] @@ -57,8 +58,8 @@ skip = [ { name = "http-body", version = "0.4.6" }, { name = "hyper", version = "0.14.28" }, - # Old version required by rand. - { name = "zerocopy", version = "0.6.6" }, + { name = "rustls-native-certs", version = "0.7.3" }, + { name = "hashbrown", version = "0.14.5" }, ] [sources] diff --git a/node/libs/crypto/src/keccak256/mod.rs b/node/libs/crypto/src/keccak256/mod.rs index 1c94c243..ce7bd753 100644 --- a/node/libs/crypto/src/keccak256/mod.rs +++ b/node/libs/crypto/src/keccak256/mod.rs @@ -7,7 +7,7 @@ mod test; pub mod testonly; /// Keccak256 hash. -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Keccak256(pub(crate) [u8; 32]); impl Keccak256 { diff --git a/node/libs/roles/src/proto/validator/keys.proto b/node/libs/roles/src/proto/validator/keys.proto index 245e0c57..3a3ffeeb 100644 --- a/node/libs/roles/src/proto/validator/keys.proto +++ b/node/libs/roles/src/proto/validator/keys.proto @@ -3,17 +3,17 @@ syntax = "proto3"; package zksync.roles.validator; message PublicKey { - // The name is wrong, it should be bls12_381. + // TODO: The name is wrong, it should be bls12_381. optional bytes bn254 = 1; // required } message Signature { - // The name is wrong, it should be bls12_381. + // TODO: The name is wrong, it should be bls12_381. optional bytes bn254 = 1; // required } message AggregateSignature { - // The name is wrong, it should be bls12_381. + // TODO: The name is wrong, it should be bls12_381. optional bytes bn254 = 1; // required } diff --git a/node/libs/roles/src/proto/validator/messages.proto b/node/libs/roles/src/proto/validator/messages.proto index 83308704..209c4889 100644 --- a/node/libs/roles/src/proto/validator/messages.proto +++ b/node/libs/roles/src/proto/validator/messages.proto @@ -39,57 +39,68 @@ message Block { message View { reserved 1,2; reserved "protocol_version","fork"; + optional GenesisHash genesis = 4; // required optional uint64 number = 3; // required; ViewNumber } message ConsensusMsg { - oneof t {// required - ReplicaPrepare replica_prepare = 1; + reserved 1, 3, 4; + reserved "replica_prepare", "leader_prepare", "leader_commit"; + + oneof t { // required ReplicaCommit replica_commit = 2; - LeaderPrepare leader_prepare = 3; - LeaderCommit leader_commit = 4; + ReplicaTimeout replica_timeout = 5; + ReplicaNewView replica_new_view = 6; + LeaderProposal leader_proposal = 7; } } -message ReplicaPrepare { +message ReplicaCommit { + optional View view = 1; // required + optional BlockHeader proposal = 2; // required +} + +message ReplicaTimeout { optional View view = 1; // required optional ReplicaCommit high_vote = 2; // optional optional CommitQC high_qc = 3; // optional } -message ReplicaCommit { - optional View view = 1; // required - optional BlockHeader proposal = 2; // required +message ReplicaNewView { + optional ProposalJustification justification = 1; // required } -message LeaderPrepare { - optional BlockHeader proposal = 1; // required - optional bytes proposal_payload = 2; // optional (depending on justification) - optional PrepareQC justification = 3; // required +message LeaderProposal { + optional bytes proposal_payload = 1; // optional (depending on justification) + optional ProposalJustification justification = 2; // required } -message LeaderCommit { - optional CommitQC justification = 1; // required +message CommitQC { + optional ReplicaCommit msg = 1; // required + optional std.BitVector signers = 2; // required + optional AggregateSignature sig = 3; // required } -message PrepareQC { +message TimeoutQC { optional View view = 4; // required - repeated ReplicaPrepare msgs = 1; // required + repeated ReplicaTimeout msgs = 1; // required repeated std.BitVector signers = 2; // required optional AggregateSignature sig = 3; // required } -message CommitQC { - optional ReplicaCommit msg = 1; // required - optional std.BitVector signers = 2; // required - optional AggregateSignature sig = 3; // required +message ProposalJustification { + oneof t { // required + CommitQC commit_qc = 1; + TimeoutQC timeout_qc = 2; + } } message Phase { - oneof t { + oneof t { // required std.Void prepare = 1; std.Void commit = 2; + std.Void timeout = 3; } } diff --git a/node/libs/roles/src/validator/conv.rs b/node/libs/roles/src/validator/conv.rs index 25f2bca3..5addcab2 100644 --- a/node/libs/roles/src/validator/conv.rs +++ b/node/libs/roles/src/validator/conv.rs @@ -1,9 +1,9 @@ use super::{ AggregateSignature, Block, BlockHeader, BlockNumber, ChainId, CommitQC, Committee, ConsensusMsg, FinalBlock, ForkNumber, Genesis, GenesisHash, GenesisRaw, Justification, - LeaderCommit, LeaderPrepare, Msg, MsgHash, NetAddress, Payload, PayloadHash, Phase, - PreGenesisBlock, PrepareQC, ProtocolVersion, PublicKey, ReplicaCommit, ReplicaPrepare, - Signature, Signed, Signers, View, ViewNumber, WeightedValidator, + LeaderProposal, Msg, MsgHash, NetAddress, Payload, PayloadHash, Phase, PreGenesisBlock, + ProposalJustification, ProtocolVersion, PublicKey, ReplicaCommit, ReplicaNewView, + ReplicaTimeout, Signature, Signed, Signers, TimeoutQC, View, ViewNumber, WeightedValidator, }; use crate::{ attester::{self, WeightedAttester}, @@ -186,12 +186,16 @@ impl ProtoFmt for ConsensusMsg { fn read(r: &Self::Proto) -> anyhow::Result { use proto::consensus_msg::T; Ok(match r.t.as_ref().context("missing")? { - T::ReplicaPrepare(r) => { - Self::ReplicaPrepare(ProtoFmt::read(r).context("ReplicaPrepare")?) - } T::ReplicaCommit(r) => Self::ReplicaCommit(ProtoFmt::read(r).context("ReplicaCommit")?), - T::LeaderPrepare(r) => Self::LeaderPrepare(ProtoFmt::read(r).context("LeaderPrepare")?), - T::LeaderCommit(r) => Self::LeaderCommit(ProtoFmt::read(r).context("LeaderCommit")?), + T::ReplicaTimeout(r) => { + Self::ReplicaTimeout(ProtoFmt::read(r).context("ReplicaTimeout")?) + } + T::ReplicaNewView(r) => { + Self::ReplicaNewView(ProtoFmt::read(r).context("ReplicaNewView")?) + } + T::LeaderProposal(r) => { + Self::LeaderProposal(ProtoFmt::read(r).context("LeaderProposal")?) + } }) } @@ -199,10 +203,10 @@ impl ProtoFmt for ConsensusMsg { use proto::consensus_msg::T; let t = match self { - Self::ReplicaPrepare(x) => T::ReplicaPrepare(x.build()), Self::ReplicaCommit(x) => T::ReplicaCommit(x.build()), - Self::LeaderPrepare(x) => T::LeaderPrepare(x.build()), - Self::LeaderCommit(x) => T::LeaderCommit(x.build()), + Self::ReplicaTimeout(x) => T::ReplicaTimeout(x.build()), + Self::ReplicaNewView(x) => T::ReplicaNewView(x.build()), + Self::LeaderProposal(x) => T::LeaderProposal(x.build()), }; Self::Proto { t: Some(t) } @@ -227,101 +231,107 @@ impl ProtoFmt for View { } } -impl ProtoFmt for ReplicaPrepare { - type Proto = proto::ReplicaPrepare; +impl ProtoFmt for ReplicaCommit { + type Proto = proto::ReplicaCommit; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { view: read_required(&r.view).context("view")?, - high_vote: read_optional(&r.high_vote).context("high_vote")?, - high_qc: read_optional(&r.high_qc).context("high_qc")?, + proposal: read_required(&r.proposal).context("proposal")?, }) } fn build(&self) -> Self::Proto { Self::Proto { view: Some(self.view.build()), - high_vote: self.high_vote.as_ref().map(ProtoFmt::build), - high_qc: self.high_qc.as_ref().map(ProtoFmt::build), + proposal: Some(self.proposal.build()), } } } -impl ProtoFmt for ReplicaCommit { - type Proto = proto::ReplicaCommit; +impl ProtoFmt for ReplicaTimeout { + type Proto = proto::ReplicaTimeout; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { view: read_required(&r.view).context("view")?, - proposal: read_required(&r.proposal).context("proposal")?, + high_vote: read_optional(&r.high_vote).context("high_vote")?, + high_qc: read_optional(&r.high_qc).context("high_qc")?, }) } fn build(&self) -> Self::Proto { Self::Proto { view: Some(self.view.build()), - proposal: Some(self.proposal.build()), + high_vote: self.high_vote.as_ref().map(ProtoFmt::build), + high_qc: self.high_qc.as_ref().map(ProtoFmt::build), } } } -impl ProtoFmt for LeaderPrepare { - type Proto = proto::LeaderPrepare; +impl ProtoFmt for ReplicaNewView { + type Proto = proto::ReplicaNewView; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { - proposal: read_required(&r.proposal).context("proposal")?, - proposal_payload: r.proposal_payload.as_ref().map(|p| Payload(p.clone())), justification: read_required(&r.justification).context("justification")?, }) } fn build(&self) -> Self::Proto { Self::Proto { - proposal: Some(self.proposal.build()), - proposal_payload: self.proposal_payload.as_ref().map(|p| p.0.clone()), justification: Some(self.justification.build()), } } } -impl ProtoFmt for LeaderCommit { - type Proto = proto::LeaderCommit; +impl ProtoFmt for LeaderProposal { + type Proto = proto::LeaderProposal; fn read(r: &Self::Proto) -> anyhow::Result { Ok(Self { + proposal_payload: r.proposal_payload.as_ref().map(|p| Payload(p.clone())), justification: read_required(&r.justification).context("justification")?, }) } fn build(&self) -> Self::Proto { Self::Proto { + proposal_payload: self.proposal_payload.as_ref().map(|p| p.0.clone()), justification: Some(self.justification.build()), } } } -impl ProtoFmt for Signers { - type Proto = zksync_protobuf::proto::std::BitVector; +impl ProtoFmt for CommitQC { + type Proto = proto::CommitQc; fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self(ProtoFmt::read(r)?)) + Ok(Self { + message: read_required(&r.msg).context("msg")?, + signers: read_required(&r.signers).context("signers")?, + signature: read_required(&r.sig).context("sig")?, + }) } fn build(&self) -> Self::Proto { - self.0.build() + Self::Proto { + msg: Some(self.message.build()), + signers: Some(self.signers.build()), + sig: Some(self.signature.build()), + } } } -impl ProtoFmt for PrepareQC { - type Proto = proto::PrepareQc; +impl ProtoFmt for TimeoutQC { + type Proto = proto::TimeoutQc; fn read(r: &Self::Proto) -> anyhow::Result { let mut map = BTreeMap::new(); for (msg, signers) in r.msgs.iter().zip(r.signers.iter()) { map.insert( - ReplicaPrepare::read(msg).context("msg")?, + ReplicaTimeout::read(msg).context("msg")?, Signers::read(signers).context("signers")?, ); } @@ -349,23 +359,38 @@ impl ProtoFmt for PrepareQC { } } -impl ProtoFmt for CommitQC { - type Proto = proto::CommitQc; +impl ProtoFmt for ProposalJustification { + type Proto = proto::ProposalJustification; fn read(r: &Self::Proto) -> anyhow::Result { - Ok(Self { - message: read_required(&r.msg).context("msg")?, - signers: read_required(&r.signers).context("signers")?, - signature: read_required(&r.sig).context("sig")?, + use proto::proposal_justification::T; + Ok(match r.t.as_ref().context("missing")? { + T::CommitQc(r) => Self::Commit(ProtoFmt::read(r).context("Commit")?), + T::TimeoutQc(r) => Self::Timeout(ProtoFmt::read(r).context("Timeout")?), }) } fn build(&self) -> Self::Proto { - Self::Proto { - msg: Some(self.message.build()), - signers: Some(self.signers.build()), - sig: Some(self.signature.build()), - } + use proto::proposal_justification::T; + + let t = match self { + Self::Commit(x) => T::CommitQc(x.build()), + Self::Timeout(x) => T::TimeoutQc(x.build()), + }; + + Self::Proto { t: Some(t) } + } +} + +impl ProtoFmt for Signers { + type Proto = zksync_protobuf::proto::std::BitVector; + + fn read(r: &Self::Proto) -> anyhow::Result { + Ok(Self(ProtoFmt::read(r)?)) + } + + fn build(&self) -> Self::Proto { + self.0.build() } } @@ -377,6 +402,7 @@ impl ProtoFmt for Phase { Ok(match required(&r.t)? { T::Prepare(_) => Self::Prepare, T::Commit(_) => Self::Commit, + T::Timeout(_) => Self::Timeout, }) } @@ -385,6 +411,7 @@ impl ProtoFmt for Phase { let t = match self { Self::Prepare => T::Prepare(zksync_protobuf::proto::std::Void {}), Self::Commit => T::Commit(zksync_protobuf::proto::std::Void {}), + Self::Timeout => T::Timeout(zksync_protobuf::proto::std::Void {}), }; Self::Proto { t: Some(t) } } diff --git a/node/libs/roles/src/validator/keys/mod.rs b/node/libs/roles/src/validator/keys/mod.rs index f8d18c94..a4816015 100644 --- a/node/libs/roles/src/validator/keys/mod.rs +++ b/node/libs/roles/src/validator/keys/mod.rs @@ -4,6 +4,8 @@ mod aggregate_signature; mod public_key; mod secret_key; mod signature; +#[cfg(test)] +mod tests; pub use aggregate_signature::AggregateSignature; pub use public_key::PublicKey; diff --git a/node/libs/roles/src/validator/keys/signature.rs b/node/libs/roles/src/validator/keys/signature.rs index 76e3419c..d5ed9583 100644 --- a/node/libs/roles/src/validator/keys/signature.rs +++ b/node/libs/roles/src/validator/keys/signature.rs @@ -19,14 +19,9 @@ impl Signature { } } -/// Proof of possession of a validator secret key. -#[derive(Clone, PartialEq, Eq)] -pub struct ProofOfPossession(pub(crate) bls12_381::ProofOfPossession); - -impl ProofOfPossession { - /// Verifies the proof against the public key. - pub fn verify(&self, pk: &PublicKey) -> anyhow::Result<()> { - self.0.verify(&pk.0) +impl fmt::Debug for Signature { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&TextFmt::encode(self)) } } @@ -39,15 +34,6 @@ impl ByteFmt for Signature { } } -impl ByteFmt for ProofOfPossession { - fn encode(&self) -> Vec { - ByteFmt::encode(&self.0) - } - fn decode(bytes: &[u8]) -> anyhow::Result { - ByteFmt::decode(bytes).map(Self) - } -} - impl TextFmt for Signature { fn encode(&self) -> String { format!( @@ -62,6 +48,32 @@ impl TextFmt for Signature { } } +/// Proof of possession of a validator secret key. +#[derive(Clone, PartialEq, Eq)] +pub struct ProofOfPossession(pub(crate) bls12_381::ProofOfPossession); + +impl ProofOfPossession { + /// Verifies the proof against the public key. + pub fn verify(&self, pk: &PublicKey) -> anyhow::Result<()> { + self.0.verify(&pk.0) + } +} + +impl fmt::Debug for ProofOfPossession { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(&TextFmt::encode(self)) + } +} + +impl ByteFmt for ProofOfPossession { + fn encode(&self) -> Vec { + ByteFmt::encode(&self.0) + } + fn decode(bytes: &[u8]) -> anyhow::Result { + ByteFmt::decode(bytes).map(Self) + } +} + impl TextFmt for ProofOfPossession { fn encode(&self) -> String { format!( @@ -75,15 +87,3 @@ impl TextFmt for ProofOfPossession { .map(Self) } } - -impl fmt::Debug for ProofOfPossession { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(&TextFmt::encode(self)) - } -} - -impl fmt::Debug for Signature { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(&TextFmt::encode(self)) - } -} diff --git a/node/libs/roles/src/validator/keys/tests.rs b/node/libs/roles/src/validator/keys/tests.rs new file mode 100644 index 00000000..b52e8778 --- /dev/null +++ b/node/libs/roles/src/validator/keys/tests.rs @@ -0,0 +1,60 @@ +use super::*; +use crate::validator::MsgHash; +use rand::Rng as _; +use std::vec; +use zksync_concurrency::ctx; + +#[test] +fn test_signature_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let msg1: MsgHash = rng.gen(); + let msg2: MsgHash = rng.gen(); + + let key1: SecretKey = rng.gen(); + let key2: SecretKey = rng.gen(); + + let sig1 = key1.sign_hash(&msg1); + + // Matching key and message. + sig1.verify_hash(&msg1, &key1.public()).unwrap(); + + // Mismatching message. + assert!(sig1.verify_hash(&msg2, &key1.public()).is_err()); + + // Mismatching key. + assert!(sig1.verify_hash(&msg1, &key2.public()).is_err()); +} + +#[test] +fn test_agg_signature_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let msg1: MsgHash = rng.gen(); + let msg2: MsgHash = rng.gen(); + + let key1: SecretKey = rng.gen(); + let key2: SecretKey = rng.gen(); + + let sig1 = key1.sign_hash(&msg1); + let sig2 = key2.sign_hash(&msg2); + + let agg_sig = AggregateSignature::aggregate(vec![&sig1, &sig2]); + + // Matching key and message. + agg_sig + .verify_hash([(msg1, &key1.public()), (msg2, &key2.public())].into_iter()) + .unwrap(); + + // Mismatching message. + assert!(agg_sig + .verify_hash([(msg2, &key1.public()), (msg1, &key2.public())].into_iter()) + .is_err()); + + // Mismatching key. + assert!(agg_sig + .verify_hash([(msg1, &key2.public()), (msg2, &key1.public())].into_iter()) + .is_err()); +} diff --git a/node/libs/roles/src/validator/messages/block.rs b/node/libs/roles/src/validator/messages/block.rs index 2fa3c765..2c0f74af 100644 --- a/node/libs/roles/src/validator/messages/block.rs +++ b/node/libs/roles/src/validator/messages/block.rs @@ -21,6 +21,23 @@ impl fmt::Debug for Payload { } } +impl Payload { + /// Hash of the payload. + pub fn hash(&self) -> PayloadHash { + PayloadHash(Keccak256::new(&self.0)) + } + + /// Returns the length of the payload. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the payload is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + /// Hash of the Payload. #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PayloadHash(pub(crate) Keccak256); @@ -44,13 +61,6 @@ impl fmt::Debug for PayloadHash { } } -impl Payload { - /// Hash of the payload. - pub fn hash(&self) -> PayloadHash { - PayloadHash(Keccak256::new(&self.0)) - } -} - /// Sequential number of the block. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BlockNumber(pub u64); @@ -102,6 +112,7 @@ impl FinalBlock { /// Creates a new finalized block. pub fn new(payload: Payload, justification: CommitQC) -> Self { assert_eq!(justification.header().payload, payload.hash()); + Self { payload, justification, diff --git a/node/libs/roles/src/validator/messages/committee.rs b/node/libs/roles/src/validator/messages/committee.rs new file mode 100644 index 00000000..1241c540 --- /dev/null +++ b/node/libs/roles/src/validator/messages/committee.rs @@ -0,0 +1,213 @@ +//! Messages related to the consensus protocol. +use super::{Signers, ViewNumber}; +use crate::validator; +use anyhow::Context; +use num_bigint::BigUint; +use std::collections::BTreeMap; +use zksync_consensus_crypto::keccak256::Keccak256; + +/// A struct that represents a set of validators. It is used to store the current validator set. +/// We represent each validator by its validator public key. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Committee { + vec: Vec, + indexes: BTreeMap, + total_weight: u64, +} + +impl std::ops::Deref for Committee { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.vec + } +} + +impl Committee { + /// Creates a new Committee from a list of validator public keys. Note that the order of the given validators + /// is NOT preserved in the committee. + pub fn new(validators: impl IntoIterator) -> anyhow::Result { + let mut map = BTreeMap::new(); + let mut total_weight: u64 = 0; + for v in validators { + anyhow::ensure!( + !map.contains_key(&v.key), + "Duplicate validator in validator Committee" + ); + anyhow::ensure!(v.weight > 0, "Validator weight has to be a positive value"); + total_weight = total_weight + .checked_add(v.weight) + .context("Sum of weights overflows in validator Committee")?; + map.insert(v.key.clone(), v); + } + anyhow::ensure!( + !map.is_empty(), + "Validator Committee must contain at least one validator" + ); + let vec: Vec<_> = map.into_values().collect(); + Ok(Self { + indexes: vec + .iter() + .enumerate() + .map(|(i, v)| (v.key.clone(), i)) + .collect(), + vec, + total_weight, + }) + } + + /// Iterates over validator keys. + pub fn keys(&self) -> impl Iterator { + self.vec.iter().map(|v| &v.key) + } + + /// Returns the number of validators. + #[allow(clippy::len_without_is_empty)] // a valid `Committee` is always non-empty by construction + pub fn len(&self) -> usize { + self.vec.len() + } + + /// Returns true if the given validator is in the validator committee. + pub fn contains(&self, validator: &validator::PublicKey) -> bool { + self.indexes.contains_key(validator) + } + + /// Get validator by its index in the committee. + pub fn get(&self, index: usize) -> Option<&WeightedValidator> { + self.vec.get(index) + } + + /// Get the index of a validator in the committee. + pub fn index(&self, validator: &validator::PublicKey) -> Option { + self.indexes.get(validator).copied() + } + + /// Computes the leader for the given view. + pub fn view_leader( + &self, + view_number: ViewNumber, + leader_selection: &LeaderSelectionMode, + ) -> validator::PublicKey { + match &leader_selection { + LeaderSelectionMode::RoundRobin => { + let index = view_number.0 as usize % self.len(); + self.get(index).unwrap().key.clone() + } + LeaderSelectionMode::Weighted => { + let eligibility = LeaderSelectionMode::leader_weighted_eligibility( + view_number.0, + self.total_weight, + ); + let mut offset = 0; + for val in &self.vec { + offset += val.weight; + if eligibility < offset { + return val.key.clone(); + } + } + unreachable!() + } + LeaderSelectionMode::Sticky(pk) => { + let index = self.index(pk).unwrap(); + self.get(index).unwrap().key.clone() + } + LeaderSelectionMode::Rota(pks) => { + let index = view_number.0 as usize % pks.len(); + let index = self.index(&pks[index]).unwrap(); + self.get(index).unwrap().key.clone() + } + } + } + + /// Signature weight threshold for this validator committee. + pub fn quorum_threshold(&self) -> u64 { + quorum_threshold(self.total_weight()) + } + + /// Signature weight threshold for this validator committee to trigger a reproposal. + pub fn subquorum_threshold(&self) -> u64 { + subquorum_threshold(self.total_weight()) + } + + /// Maximal weight of faulty replicas allowed in this validator committee. + pub fn max_faulty_weight(&self) -> u64 { + max_faulty_weight(self.total_weight()) + } + + /// Compute the sum of signers weights. + /// Panics if signers length does not match the number of validators in committee + pub fn weight(&self, signers: &Signers) -> u64 { + assert_eq!(self.vec.len(), signers.len()); + self.vec + .iter() + .enumerate() + .filter(|(i, _)| signers.0[*i]) + .map(|(_, v)| v.weight) + .sum() + } + + /// Sum of all validators' weight in the committee + pub fn total_weight(&self) -> u64 { + self.total_weight + } +} + +/// Validator representation inside a Committee. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WeightedValidator { + /// Validator key + pub key: validator::PublicKey, + /// Validator weight inside the Committee. + pub weight: Weight, +} + +/// Voting weight. +pub type Weight = u64; + +/// The mode used for selecting leader for a given view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum LeaderSelectionMode { + /// Select in a round-robin fashion, based on validators' index within the set. + RoundRobin, + /// Select based on a sticky assignment to a specific validator. + Sticky(validator::PublicKey), + /// Select pseudo-randomly, based on validators' weights. + Weighted, + /// Select on a rotation of specific validator keys. + Rota(Vec), +} + +impl LeaderSelectionMode { + /// Calculates the pseudo-random eligibility of a leader based on the input and total weight. + pub fn leader_weighted_eligibility(input: u64, total_weight: u64) -> u64 { + let input_bytes = input.to_be_bytes(); + let hash = Keccak256::new(&input_bytes); + let hash_big = BigUint::from_bytes_be(hash.as_bytes()); + let total_weight_big = BigUint::from(total_weight); + let ret_big = hash_big % total_weight_big; + // Assumes that `ret_big` does not exceed 64 bits due to the modulo operation with a 64 bits-capped value. + ret_big.to_u64_digits()[0] + } +} + +/// Calculate the maximum allowed weight for faulty replicas, for a given total weight. +pub fn max_faulty_weight(total_weight: u64) -> u64 { + // Calculate the allowed maximum weight of faulty replicas. We want the following relationship to hold: + // n = 5*f + 1 + // for n total weight and f faulty weight. This results in the following formula for the maximum + // weight of faulty replicas: + // f = floor((n - 1) / 5) + (total_weight - 1) / 5 +} + +/// Calculate the consensus quorum threshold, the minimum votes' weight necessary to finalize a block, +/// for a given committee total weight. +pub fn quorum_threshold(total_weight: u64) -> u64 { + total_weight - max_faulty_weight(total_weight) +} + +/// Calculate the consensus subquorum threshold, the minimum votes' weight necessary to trigger a reproposal, +/// for a given committee total weight. +pub fn subquorum_threshold(total_weight: u64) -> u64 { + total_weight - 3 * max_faulty_weight(total_weight) +} diff --git a/node/libs/roles/src/validator/messages/consensus.rs b/node/libs/roles/src/validator/messages/consensus.rs index f9895f72..1ff67b4d 100644 --- a/node/libs/roles/src/validator/messages/consensus.rs +++ b/node/libs/roles/src/validator/messages/consensus.rs @@ -1,393 +1,54 @@ //! Messages related to the consensus protocol. -use super::{BlockNumber, LeaderCommit, LeaderPrepare, Msg, ReplicaCommit, ReplicaPrepare}; -use crate::{attester, validator}; -use anyhow::Context; +use super::{ + Genesis, GenesisHash, LeaderProposal, Msg, ReplicaCommit, ReplicaNewView, ReplicaTimeout, +}; use bit_vec::BitVec; -use num_bigint::BigUint; -use std::{collections::BTreeMap, fmt, hash::Hash}; -use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt, Text, TextFmt}; +use std::{fmt, hash::Hash}; use zksync_consensus_utils::enum_util::{BadVariantError, Variant}; -/// Version of the consensus algorithm that the validator is using. -/// It allows to prevent misinterpretation of messages signed by validators -/// using different versions of the binaries. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ProtocolVersion(pub u32); - -impl ProtocolVersion { - /// 0 - development version; deprecated. - /// 1 - development version - pub const CURRENT: Self = Self(1); - - /// Returns the integer corresponding to this version. - pub fn as_u32(self) -> u32 { - self.0 - } - - /// Checks protocol version compatibility. - pub fn compatible(&self, other: &ProtocolVersion) -> bool { - // Currently using comparison. - // This can be changed later to apply a minimum supported version. - self.0 == other.0 - } -} - -impl TryFrom for ProtocolVersion { - type Error = anyhow::Error; - - fn try_from(value: u32) -> Result { - // Currently, consensus doesn't define restrictions on the possible version. Unsupported - // versions are filtered out on the BFT actor level instead. - Ok(Self(value)) - } -} - -/// Number of the fork. Newer fork has higher number. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ForkNumber(pub u64); - -impl ForkNumber { - /// Next fork number. - pub fn next(self) -> Self { - Self(self.0 + 1) - } -} - -/// The mode used for selecting leader for a given view. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum LeaderSelectionMode { - /// Select in a round-robin fashion, based on validators' index within the set. - RoundRobin, - - /// Select based on a sticky assignment to a specific validator. - Sticky(validator::PublicKey), - - /// Select pseudo-randomly, based on validators' weights. - Weighted, - - /// Select on a rotation of specific validator keys. - Rota(Vec), -} - -/// Calculates the pseudo-random eligibility of a leader based on the input and total weight. -pub(crate) fn leader_weighted_eligibility(input: u64, total_weight: u64) -> u64 { - let input_bytes = input.to_be_bytes(); - let hash = Keccak256::new(&input_bytes); - let hash_big = BigUint::from_bytes_be(hash.as_bytes()); - let total_weight_big = BigUint::from(total_weight); - let ret_big = hash_big % total_weight_big; - // Assumes that `ret_big` does not exceed 64 bits due to the modulo operation with a 64 bits-capped value. - ret_big.to_u64_digits()[0] -} - -/// A struct that represents a set of validators. It is used to store the current validator set. -/// We represent each validator by its validator public key. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Committee { - vec: Vec, - indexes: BTreeMap, - total_weight: u64, -} - -impl std::ops::Deref for Committee { - type Target = Vec; - - fn deref(&self) -> &Self::Target { - &self.vec - } -} - -impl Committee { - /// Creates a new Committee from a list of validator public keys. - pub fn new(validators: impl IntoIterator) -> anyhow::Result { - let mut map = BTreeMap::new(); - let mut total_weight: u64 = 0; - for v in validators { - anyhow::ensure!( - !map.contains_key(&v.key), - "Duplicate validator in validator Committee" - ); - anyhow::ensure!(v.weight > 0, "Validator weight has to be a positive value"); - total_weight = total_weight - .checked_add(v.weight) - .context("Sum of weights overflows in validator Committee")?; - map.insert(v.key.clone(), v); - } - anyhow::ensure!( - !map.is_empty(), - "Validator Committee must contain at least one validator" - ); - let vec: Vec<_> = map.into_values().collect(); - Ok(Self { - indexes: vec - .iter() - .enumerate() - .map(|(i, v)| (v.key.clone(), i)) - .collect(), - vec, - total_weight, - }) - } - - /// Iterates over validator keys. - pub fn keys(&self) -> impl Iterator { - self.vec.iter().map(|v| &v.key) - } - - /// Returns the number of validators. - #[allow(clippy::len_without_is_empty)] // a valid `Committee` is always non-empty by construction - pub fn len(&self) -> usize { - self.vec.len() - } - - /// Returns true if the given validator is in the validator committee. - pub fn contains(&self, validator: &validator::PublicKey) -> bool { - self.indexes.contains_key(validator) - } - - /// Get validator by its index in the committee. - pub fn get(&self, index: usize) -> Option<&WeightedValidator> { - self.vec.get(index) - } - - /// Get the index of a validator in the committee. - pub fn index(&self, validator: &validator::PublicKey) -> Option { - self.indexes.get(validator).copied() - } - - /// Computes the leader for the given view. - pub fn view_leader( - &self, - view_number: ViewNumber, - leader_selection: &LeaderSelectionMode, - ) -> validator::PublicKey { - match &leader_selection { - LeaderSelectionMode::RoundRobin => { - let index = view_number.0 as usize % self.len(); - self.get(index).unwrap().key.clone() - } - LeaderSelectionMode::Weighted => { - let eligibility = leader_weighted_eligibility(view_number.0, self.total_weight); - let mut offset = 0; - for val in &self.vec { - offset += val.weight; - if eligibility < offset { - return val.key.clone(); - } - } - unreachable!() - } - LeaderSelectionMode::Sticky(pk) => { - let index = self.index(pk).unwrap(); - self.get(index).unwrap().key.clone() - } - LeaderSelectionMode::Rota(pks) => { - let index = view_number.0 as usize % pks.len(); - let index = self.index(&pks[index]).unwrap(); - self.get(index).unwrap().key.clone() - } - } - } - - /// Signature weight threshold for this validator committee. - pub fn threshold(&self) -> u64 { - threshold(self.total_weight()) - } - - /// Maximal weight of faulty replicas allowed in this validator committee. - pub fn max_faulty_weight(&self) -> u64 { - max_faulty_weight(self.total_weight()) - } - - /// Compute the sum of signers weights. - /// Panics if signers length does not match the number of validators in committee - pub fn weight(&self, signers: &Signers) -> u64 { - assert_eq!(self.vec.len(), signers.len()); - self.vec - .iter() - .enumerate() - .filter(|(i, _)| signers.0[*i]) - .map(|(_, v)| v.weight) - .sum() - } - - /// Sum of all validators' weight in the committee - pub fn total_weight(&self) -> u64 { - self.total_weight - } -} - -/// Calculate the consensus threshold, the minimum votes' weight for any consensus action to be valid, -/// for a given committee total weight. -pub fn threshold(total_weight: u64) -> u64 { - total_weight - max_faulty_weight(total_weight) -} - -/// Calculate the maximum allowed weight for faulty replicas, for a given total weight. -pub fn max_faulty_weight(total_weight: u64) -> u64 { - // Calculate the allowed maximum weight of faulty replicas. We want the following relationship to hold: - // n = 5*f + 1 - // for n total weight and f faulty weight. This results in the following formula for the maximum - // weight of faulty replicas: - // f = floor((n - 1) / 5) - (total_weight - 1) / 5 -} - -/// Ethereum CHAIN_ID -/// `https://github.com/ethereum/EIPs/blob/master/EIPS/eip-155.md` -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct ChainId(pub u64); - -/// Genesis of the blockchain, unique for each blockchain instance. -#[derive(Debug, Clone, PartialEq)] -pub struct GenesisRaw { - /// ID of the blockchain. - pub chain_id: ChainId, - /// Number of the fork. Should be incremented every time the genesis is updated, - /// i.e. whenever a hard fork is performed. - pub fork_number: ForkNumber, - /// Protocol version used by this fork. - pub protocol_version: ProtocolVersion, - /// First block of a fork. - pub first_block: BlockNumber, - /// Set of validators of the chain. - pub validators: Committee, - /// Set of attesters of the chain. - pub attesters: Option, - /// The mode used for selecting leader for a given view. - pub leader_selection: LeaderSelectionMode, -} - -/// Hash of the genesis specification. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct GenesisHash(pub(crate) Keccak256); - -impl GenesisRaw { - /// Constructs Genesis with cached hash. - pub fn with_hash(self) -> Genesis { - let hash = GenesisHash(Keccak256::new(&zksync_protobuf::canonical(&self))); - Genesis(self, hash) - } -} - -impl TextFmt for GenesisHash { - fn decode(text: Text) -> anyhow::Result { - text.strip("genesis_hash:keccak256:")? - .decode_hex() - .map(Self) - } - - fn encode(&self) -> String { - format!( - "genesis_hash:keccak256:{}", - hex::encode(ByteFmt::encode(&self.0)) - ) - } -} - -impl fmt::Debug for GenesisHash { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.write_str(&TextFmt::encode(self)) - } -} - -/// Genesis with cached hash. -#[derive(Clone)] -pub struct Genesis(GenesisRaw, GenesisHash); - -impl std::ops::Deref for Genesis { - type Target = GenesisRaw; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl PartialEq for Genesis { - fn eq(&self, other: &Self) -> bool { - self.1 == other.1 - } -} - -impl fmt::Debug for Genesis { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(fmt) - } -} - -impl Genesis { - /// Verifies correctness. - pub fn verify(&self) -> anyhow::Result<()> { - if let LeaderSelectionMode::Sticky(pk) = &self.leader_selection { - if self.validators.index(pk).is_none() { - anyhow::bail!("leader_selection sticky mode public key is not in committee"); - } - } else if let LeaderSelectionMode::Rota(pks) = &self.leader_selection { - for pk in pks { - if self.validators.index(pk).is_none() { - anyhow::bail!( - "leader_selection rota mode public key is not in committee: {pk:?}" - ); - } - } - } - - Ok(()) - } - - /// Computes the leader for the given view. - pub fn view_leader(&self, view: ViewNumber) -> validator::PublicKey { - self.validators.view_leader(view, &self.leader_selection) - } - - /// Hash of the genesis. - pub fn hash(&self) -> GenesisHash { - self.1 - } -} - /// Consensus messages. #[allow(missing_docs)] #[derive(Clone, Debug, PartialEq, Eq)] pub enum ConsensusMsg { - ReplicaPrepare(ReplicaPrepare), + LeaderProposal(LeaderProposal), ReplicaCommit(ReplicaCommit), - LeaderPrepare(LeaderPrepare), - LeaderCommit(LeaderCommit), + ReplicaNewView(ReplicaNewView), + ReplicaTimeout(ReplicaTimeout), } impl ConsensusMsg { /// ConsensusMsg variant name. pub fn label(&self) -> &'static str { match self { - Self::ReplicaPrepare(_) => "ReplicaPrepare", + Self::LeaderProposal(_) => "LeaderProposal", Self::ReplicaCommit(_) => "ReplicaCommit", - Self::LeaderPrepare(_) => "LeaderPrepare", - Self::LeaderCommit(_) => "LeaderCommit", + Self::ReplicaNewView(_) => "ReplicaNewView", + Self::ReplicaTimeout(_) => "ReplicaTimeout", } } /// View of this message. - pub fn view(&self) -> &View { + pub fn view(&self) -> View { match self { - Self::ReplicaPrepare(m) => &m.view, - Self::ReplicaCommit(m) => &m.view, - Self::LeaderPrepare(m) => m.view(), - Self::LeaderCommit(m) => m.view(), + Self::LeaderProposal(msg) => msg.view(), + Self::ReplicaCommit(msg) => msg.view, + Self::ReplicaNewView(msg) => msg.view(), + Self::ReplicaTimeout(msg) => msg.view, } } /// Hash of the genesis that defines the chain. - pub fn genesis(&self) -> &GenesisHash { - &self.view().genesis + pub fn genesis(&self) -> GenesisHash { + self.view().genesis } } -impl Variant for ReplicaPrepare { +impl Variant for LeaderProposal { fn insert(self) -> Msg { - ConsensusMsg::ReplicaPrepare(self).insert() + ConsensusMsg::LeaderProposal(self).insert() } fn extract(msg: Msg) -> Result { - let ConsensusMsg::ReplicaPrepare(this) = Variant::extract(msg)? else { + let ConsensusMsg::LeaderProposal(this) = Variant::extract(msg)? else { return Err(BadVariantError); }; Ok(this) @@ -406,24 +67,24 @@ impl Variant for ReplicaCommit { } } -impl Variant for LeaderPrepare { +impl Variant for ReplicaNewView { fn insert(self) -> Msg { - ConsensusMsg::LeaderPrepare(self).insert() + ConsensusMsg::ReplicaNewView(self).insert() } fn extract(msg: Msg) -> Result { - let ConsensusMsg::LeaderPrepare(this) = Variant::extract(msg)? else { + let ConsensusMsg::ReplicaNewView(this) = Variant::extract(msg)? else { return Err(BadVariantError); }; Ok(this) } } -impl Variant for LeaderCommit { +impl Variant for ReplicaTimeout { fn insert(self) -> Msg { - ConsensusMsg::LeaderCommit(self).insert() + ConsensusMsg::ReplicaTimeout(self).insert() } fn extract(msg: Msg) -> Result { - let ConsensusMsg::LeaderCommit(this) = Variant::extract(msg)? else { + let ConsensusMsg::ReplicaTimeout(this) = Variant::extract(msg)? else { return Err(BadVariantError); }; Ok(this) @@ -431,7 +92,7 @@ impl Variant for LeaderCommit { } /// View specification. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct View { /// Genesis of the chain this view belongs to. pub genesis: GenesisHash, @@ -445,6 +106,44 @@ impl View { anyhow::ensure!(self.genesis == genesis.hash(), "genesis mismatch"); Ok(()) } + + /// Increments the view number. + pub fn next(self) -> Self { + Self { + genesis: self.genesis, + number: ViewNumber(self.number.0 + 1), + } + } + + /// Decrements the view number. + pub fn prev(self) -> Option { + self.number.prev().map(|number| Self { + genesis: self.genesis, + number, + }) + } +} + +/// A struct that represents a view number. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ViewNumber(pub u64); + +impl ViewNumber { + /// Get the next view number. + pub fn next(self) -> Self { + Self(self.0 + 1) + } + + /// Get the previous view number. + pub fn prev(self) -> Option { + self.0.checked_sub(1).map(Self) + } +} + +impl fmt::Display for ViewNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, formatter) + } } /// Struct that represents a bit map of validators. We use it to compactly store @@ -453,7 +152,8 @@ impl View { pub struct Signers(pub BitVec); impl Signers { - /// Constructs an empty signers set. + /// Constructs a new Signers bitmap with the given number of validators. All + /// bits are set to false. pub fn new(n: usize) -> Self { Self(BitVec::from_elem(n, false)) } @@ -496,39 +196,11 @@ impl std::ops::BitAnd for &Signers { } } -/// A struct that represents a view number. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ViewNumber(pub u64); - -impl ViewNumber { - /// Get the next view number. - pub fn next(self) -> Self { - Self(self.0 + 1) - } -} - -impl fmt::Display for ViewNumber { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, formatter) - } -} - /// An enum that represents the current phase of the consensus. #[allow(missing_docs)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Phase { Prepare, Commit, + Timeout, } - -/// Validator representation inside a Committee. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WeightedValidator { - /// Validator key - pub key: validator::PublicKey, - /// Validator weight inside the Committee. - pub weight: Weight, -} - -/// Voting weight; -pub type Weight = u64; diff --git a/node/libs/roles/src/validator/messages/genesis.rs b/node/libs/roles/src/validator/messages/genesis.rs new file mode 100644 index 00000000..9f126409 --- /dev/null +++ b/node/libs/roles/src/validator/messages/genesis.rs @@ -0,0 +1,162 @@ +//! Messages related to the consensus protocol. +use super::{BlockNumber, LeaderSelectionMode, ViewNumber}; +use crate::{attester, validator}; +use std::{fmt, hash::Hash}; +use zksync_consensus_crypto::{keccak256::Keccak256, ByteFmt, Text, TextFmt}; + +/// Genesis of the blockchain, unique for each blockchain instance. +#[derive(Debug, Clone, PartialEq)] +pub struct GenesisRaw { + /// ID of the blockchain. + pub chain_id: ChainId, + /// Number of the fork. Should be incremented every time the genesis is updated, + /// i.e. whenever a hard fork is performed. + pub fork_number: ForkNumber, + /// Protocol version used by this fork. + pub protocol_version: ProtocolVersion, + /// First block of a fork. + pub first_block: BlockNumber, + /// Set of validators of the chain. + pub validators: validator::Committee, + /// Set of attesters of the chain. + pub attesters: Option, + /// The mode used for selecting leader for a given view. + pub leader_selection: LeaderSelectionMode, +} + +impl GenesisRaw { + /// Constructs Genesis with cached hash. + pub fn with_hash(self) -> Genesis { + let hash = GenesisHash(Keccak256::new(&zksync_protobuf::canonical(&self))); + Genesis(self, hash) + } +} + +/// Hash of the genesis specification. +#[derive(Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct GenesisHash(pub(crate) Keccak256); + +impl TextFmt for GenesisHash { + fn decode(text: Text) -> anyhow::Result { + text.strip("genesis_hash:keccak256:")? + .decode_hex() + .map(Self) + } + + fn encode(&self) -> String { + format!( + "genesis_hash:keccak256:{}", + hex::encode(ByteFmt::encode(&self.0)) + ) + } +} + +impl fmt::Debug for GenesisHash { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.write_str(&TextFmt::encode(self)) + } +} + +/// Genesis with cached hash. +#[derive(Clone)] +pub struct Genesis(pub(crate) GenesisRaw, pub(crate) GenesisHash); + +impl std::ops::Deref for Genesis { + type Target = GenesisRaw; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl PartialEq for Genesis { + fn eq(&self, other: &Self) -> bool { + self.1 == other.1 + } +} + +impl fmt::Debug for Genesis { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(fmt) + } +} + +impl Genesis { + /// Verifies correctness. + pub fn verify(&self) -> anyhow::Result<()> { + if let LeaderSelectionMode::Sticky(pk) = &self.leader_selection { + if self.validators.index(pk).is_none() { + anyhow::bail!("leader_selection sticky mode public key is not in committee"); + } + } else if let LeaderSelectionMode::Rota(pks) = &self.leader_selection { + for pk in pks { + if self.validators.index(pk).is_none() { + anyhow::bail!( + "leader_selection rota mode public key is not in committee: {pk:?}" + ); + } + } + } + + Ok(()) + } + + /// Computes the leader for the given view. + pub fn view_leader(&self, view: ViewNumber) -> validator::PublicKey { + self.validators.view_leader(view, &self.leader_selection) + } + + /// Hash of the genesis. + pub fn hash(&self) -> GenesisHash { + self.1 + } +} + +/// Version of the consensus algorithm that the validator is using. +/// It allows to prevent misinterpretation of messages signed by validators +/// using different versions of the binaries. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProtocolVersion(pub u32); + +impl ProtocolVersion { + /// 0 - development version; deprecated. + /// 1 - development version + pub const CURRENT: Self = Self(1); + + /// Returns the integer corresponding to this version. + pub fn as_u32(self) -> u32 { + self.0 + } + + /// Checks protocol version compatibility. + pub fn compatible(&self, other: &ProtocolVersion) -> bool { + // Currently using comparison. + // This can be changed later to apply a minimum supported version. + self.0 == other.0 + } +} + +impl TryFrom for ProtocolVersion { + type Error = anyhow::Error; + + fn try_from(value: u32) -> Result { + // Currently, consensus doesn't define restrictions on the possible version. Unsupported + // versions are filtered out on the BFT actor level instead. + Ok(Self(value)) + } +} + +/// Number of the fork. Newer fork has higher number. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ForkNumber(pub u64); + +impl ForkNumber { + /// Next fork number. + pub fn next(self) -> Self { + Self(self.0 + 1) + } +} + +/// Ethereum CHAIN_ID +/// `https://github.com/ethereum/EIPs/blob/master/EIPS/eip-155.md` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ChainId(pub u64); diff --git a/node/libs/roles/src/validator/messages/leader_commit.rs b/node/libs/roles/src/validator/messages/leader_commit.rs deleted file mode 100644 index bbcedf1b..00000000 --- a/node/libs/roles/src/validator/messages/leader_commit.rs +++ /dev/null @@ -1,150 +0,0 @@ -use super::{BlockHeader, Genesis, ReplicaCommit, ReplicaCommitVerifyError, Signed, Signers, View}; -use crate::validator; - -/// A Commit message from a leader. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LeaderCommit { - /// The CommitQC that justifies the message from the leader. - pub justification: CommitQC, -} - -impl LeaderCommit { - /// Verifies LeaderCommit. - pub fn verify(&self, genesis: &Genesis) -> Result<(), CommitQCVerifyError> { - self.justification.verify(genesis) - } - - /// View of this message. - pub fn view(&self) -> &View { - self.justification.view() - } -} - -/// A Commit Quorum Certificate. It is an aggregate of signed replica Commit messages. -/// The Quorum Certificate is supposed to be over identical messages, so we only need one message. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct CommitQC { - /// The replica Commit message that the QC is for. - pub message: ReplicaCommit, - /// The validators that signed this message. - pub signers: Signers, - /// The aggregate signature of the signed replica messages. - pub signature: validator::AggregateSignature, -} - -/// Error returned by `CommitQc::verify()`. -#[derive(thiserror::Error, Debug)] -pub enum CommitQCVerifyError { - /// Invalid message. - #[error(transparent)] - InvalidMessage(#[from] ReplicaCommitVerifyError), - /// Bad signer set. - #[error("signers set doesn't match genesis")] - BadSignersSet, - /// Weight not reached. - #[error("Signers have not reached threshold weight: got {got}, want {want}")] - NotEnoughSigners { - /// Got weight. - got: u64, - /// Want weight. - want: u64, - }, - /// Bad signature. - #[error("bad signature: {0:#}")] - BadSignature(#[source] anyhow::Error), -} - -/// Error returned by `CommitQC::add()`. -#[derive(thiserror::Error, Debug)] -pub enum CommitQCAddError { - /// Inconsistent messages. - #[error("Trying to add signature for a different message")] - InconsistentMessages, - /// Signer not present in the committee. - #[error("Signer not in committee: {signer:?}")] - SignerNotInCommittee { - /// Signer of the message. - signer: Box, - }, - /// Message already present in CommitQC. - #[error("Message already signed for CommitQC")] - Exists, -} - -impl CommitQC { - /// Header of the certified block. - pub fn header(&self) -> &BlockHeader { - &self.message.proposal - } - - /// View of this QC. - pub fn view(&self) -> &View { - &self.message.view - } - - /// Create a new empty instance for a given `ReplicaCommit` message and a validator set size. - pub fn new(message: ReplicaCommit, genesis: &Genesis) -> Self { - Self { - message, - signers: Signers::new(genesis.validators.len()), - signature: validator::AggregateSignature::default(), - } - } - - /// Add a validator's signature. - /// Signature is assumed to be already verified. - pub fn add( - &mut self, - msg: &Signed, - genesis: &Genesis, - ) -> Result<(), CommitQCAddError> { - use CommitQCAddError as Error; - if self.message != msg.msg { - return Err(Error::InconsistentMessages); - }; - let Some(i) = genesis.validators.index(&msg.key) else { - return Err(Error::SignerNotInCommittee { - signer: Box::new(msg.key.clone()), - }); - }; - if self.signers.0[i] { - return Err(Error::Exists); - }; - self.signers.0.set(i, true); - self.signature.add(&msg.sig); - Ok(()) - } - - /// Verifies the signature of the CommitQC. - pub fn verify(&self, genesis: &Genesis) -> Result<(), CommitQCVerifyError> { - use CommitQCVerifyError as Error; - self.message - .verify(genesis) - .map_err(Error::InvalidMessage)?; - if self.signers.len() != genesis.validators.len() { - return Err(Error::BadSignersSet); - } - - // Verify the signers' weight is enough. - let weight = genesis.validators.weight(&self.signers); - let threshold = genesis.validators.threshold(); - if weight < threshold { - return Err(Error::NotEnoughSigners { - got: weight, - want: threshold, - }); - } - - // Now we can verify the signature. - let messages_and_keys = genesis - .validators - .keys() - .enumerate() - .filter(|(i, _)| self.signers.0[*i]) - .map(|(_, pk)| (self.message.clone(), pk)); - - self.signature - .verify_messages(messages_and_keys) - .map_err(Error::BadSignature) - } -} diff --git a/node/libs/roles/src/validator/messages/leader_prepare.rs b/node/libs/roles/src/validator/messages/leader_prepare.rs deleted file mode 100644 index b5b9825f..00000000 --- a/node/libs/roles/src/validator/messages/leader_prepare.rs +++ /dev/null @@ -1,300 +0,0 @@ -use super::{ - BlockHeader, BlockNumber, CommitQC, Genesis, Payload, ReplicaPrepare, - ReplicaPrepareVerifyError, Signed, Signers, View, -}; -use crate::validator; -use std::collections::{BTreeMap, HashMap}; - -/// A quorum certificate of replica Prepare messages. Since not all Prepare messages are -/// identical (they have different high blocks and high QCs), we need to keep the high blocks -/// and high QCs in a map. We can still aggregate the signatures though. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PrepareQC { - /// View of this QC. - pub view: View, - /// Map from replica Prepare messages to the validators that signed them. - pub map: BTreeMap, - /// Aggregate signature of the replica Prepare messages. - pub signature: validator::AggregateSignature, -} - -/// Error returned by `PrepareQC::verify()`. -#[derive(thiserror::Error, Debug)] -pub enum PrepareQCVerifyError { - /// Bad view. - #[error("view: {0:#}")] - View(anyhow::Error), - /// Inconsistent views. - #[error("inconsistent views of signed messages")] - InconsistentViews, - /// Invalid message. - #[error("msg[{0}]: {1:#}")] - InvalidMessage(usize, ReplicaPrepareVerifyError), - /// Bad message format. - #[error(transparent)] - BadFormat(anyhow::Error), - /// Weight not reached. - #[error("Signers have not reached threshold weight: got {got}, want {want}")] - NotEnoughSigners { - /// Got weight. - got: u64, - /// Want weight. - want: u64, - }, - /// Bad signature. - #[error("bad signature: {0:#}")] - BadSignature(#[source] anyhow::Error), -} - -/// Error returned by `PrepareQC::add()`. -#[derive(thiserror::Error, Debug)] -pub enum PrepareQCAddError { - /// Inconsistent views. - #[error("Trying to add a message from a different view")] - InconsistentViews, - /// Signer not present in the committee. - #[error("Signer not in committee: {signer:?}")] - SignerNotInCommittee { - /// Signer of the message. - signer: Box, - }, - /// Message already present in PrepareQC. - #[error("Message already signed for PrepareQC")] - Exists, -} - -impl PrepareQC { - /// Create a new empty instance for a given `ReplicaCommit` message and a validator set size. - pub fn new(view: View) -> Self { - Self { - view, - map: BTreeMap::new(), - signature: validator::AggregateSignature::default(), - } - } - - /// Get the highest block voted and check if there's a quorum of votes for it. To have a quorum - /// in this situation, we require 2*f+1 votes, where f is the maximum number of faulty replicas. - /// Note that it is possible to have 2 quorums: vote A and vote B, each with >2f weight, in a single - /// PrepareQC (even in the unweighted case, because QC contains n-f signatures, not 4f+1). In such a - /// situation we say that there is no high vote. - pub fn high_vote(&self, genesis: &Genesis) -> Option { - let mut count: HashMap<_, u64> = HashMap::new(); - for (msg, signers) in &self.map { - if let Some(v) = &msg.high_vote { - *count.entry(v.proposal).or_default() += genesis.validators.weight(signers); - } - } - - let min = 2 * genesis.validators.max_faulty_weight() + 1; - let mut high_votes: Vec<_> = count.into_iter().filter(|x| x.1 >= min).collect(); - - if high_votes.len() == 1 { - high_votes.pop().map(|x| x.0) - } else { - None - } - } - - /// Get the highest CommitQC. - pub fn high_qc(&self) -> Option<&CommitQC> { - self.map - .keys() - .filter_map(|m| m.high_qc.as_ref()) - .max_by_key(|qc| qc.view().number) - } - - /// Add a validator's signed message. - /// Message is assumed to be already verified. - // TODO: verify the message inside instead. - pub fn add( - &mut self, - msg: &Signed, - genesis: &Genesis, - ) -> Result<(), PrepareQCAddError> { - use PrepareQCAddError as Error; - if msg.msg.view != self.view { - return Err(Error::InconsistentViews); - } - let Some(i) = genesis.validators.index(&msg.key) else { - return Err(Error::SignerNotInCommittee { - signer: Box::new(msg.key.clone()), - }); - }; - if self.map.values().any(|s| s.0[i]) { - return Err(Error::Exists); - }; - let e = self - .map - .entry(msg.msg.clone()) - .or_insert_with(|| Signers::new(genesis.validators.len())); - e.0.set(i, true); - self.signature.add(&msg.sig); - Ok(()) - } - - /// Verifies the integrity of the PrepareQC. - pub fn verify(&self, genesis: &Genesis) -> Result<(), PrepareQCVerifyError> { - use PrepareQCVerifyError as Error; - self.view.verify(genesis).map_err(Error::View)?; - let mut sum = Signers::new(genesis.validators.len()); - - // Check the ReplicaPrepare messages. - for (i, (msg, signers)) in self.map.iter().enumerate() { - if msg.view != self.view { - return Err(Error::InconsistentViews); - } - if signers.len() != sum.len() { - return Err(Error::BadFormat(anyhow::format_err!( - "msg[{i}].signers has wrong length" - ))); - } - if signers.is_empty() { - return Err(Error::BadFormat(anyhow::format_err!( - "msg[{i}] has no signers assigned" - ))); - } - if !(&sum & signers).is_empty() { - return Err(Error::BadFormat(anyhow::format_err!( - "overlapping signature sets for different messages" - ))); - } - msg.verify(genesis) - .map_err(|err| Error::InvalidMessage(i, err))?; - sum |= signers; - } - - // Verify the signers' weight is enough. - let weight = genesis.validators.weight(&sum); - let threshold = genesis.validators.threshold(); - if weight < threshold { - return Err(Error::NotEnoughSigners { - got: weight, - want: threshold, - }); - } - // Now we can verify the signature. - let messages_and_keys = self.map.clone().into_iter().flat_map(|(msg, signers)| { - genesis - .validators - .keys() - .enumerate() - .filter(|(i, _)| signers.0[*i]) - .map(|(_, pk)| (msg.clone(), pk)) - .collect::>() - }); - // TODO(gprusak): This reaggregating is suboptimal. - self.signature - .verify_messages(messages_and_keys) - .map_err(Error::BadSignature) - } - - /// Calculates the weight of current PrepareQC signing validators - pub fn weight(&self, committee: &validator::Committee) -> u64 { - self.map - .values() - .map(|signers| committee.weight(signers)) - .sum() - } -} - -/// A Prepare message from a leader. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LeaderPrepare { - /// The header of the block that the leader is proposing. - pub proposal: BlockHeader, - /// Payload of the block that the leader is proposing. - /// `None` iff this is a reproposal. - pub proposal_payload: Option, - /// The PrepareQC that justifies this proposal from the leader. - pub justification: PrepareQC, -} - -/// Error returned by `LeaderPrepare::verify()`. -#[derive(thiserror::Error, Debug)] -pub enum LeaderPrepareVerifyError { - /// Justification - #[error("justification: {0:#}")] - Justification(PrepareQCVerifyError), - /// Bad block number. - #[error("bad block number: got {got:?}, want {want:?}")] - BadBlockNumber { - /// Correct proposal number. - want: BlockNumber, - /// Received proposal number. - got: BlockNumber, - }, - /// New block proposal when the previous proposal was not finalized. - #[error("new block proposal when the previous proposal was not finalized")] - ProposalWhenPreviousNotFinalized, - /// Mismatched payload. - #[error("block proposal with mismatched payload")] - ProposalMismatchedPayload, - /// Re-proposal without quorum. - #[error("block re-proposal without quorum for the re-proposal")] - ReproposalWithoutQuorum, - /// Re-proposal when the previous proposal was finalized. - #[error("block re-proposal when the previous proposal was finalized")] - ReproposalWhenFinalized, - /// Reproposed a bad block. - #[error("Reproposed a bad block")] - ReproposalBadBlock, -} - -impl LeaderPrepare { - /// View of the message. - pub fn view(&self) -> &View { - &self.justification.view - } - - /// Verifies LeaderPrepare. - pub fn verify(&self, genesis: &Genesis) -> Result<(), LeaderPrepareVerifyError> { - use LeaderPrepareVerifyError as Error; - self.justification - .verify(genesis) - .map_err(Error::Justification)?; - let high_vote = self.justification.high_vote(genesis); - let high_qc = self.justification.high_qc(); - - // Check that the proposal is valid. - match &self.proposal_payload { - // The leader proposed a new block. - Some(payload) => { - // Check that payload matches the header - if self.proposal.payload != payload.hash() { - return Err(Error::ProposalMismatchedPayload); - } - // Check that we finalized the previous block. - if high_vote.is_some() - && high_vote.as_ref() != high_qc.map(|qc| &qc.message.proposal) - { - return Err(Error::ProposalWhenPreviousNotFinalized); - } - let want_number = match high_qc { - Some(qc) => qc.header().number.next(), - None => genesis.first_block, - }; - if self.proposal.number != want_number { - return Err(Error::BadBlockNumber { - got: self.proposal.number, - want: want_number, - }); - } - } - None => { - let Some(high_vote) = &high_vote else { - return Err(Error::ReproposalWithoutQuorum); - }; - if let Some(high_qc) = &high_qc { - if high_vote.number == high_qc.header().number { - return Err(Error::ReproposalWhenFinalized); - } - } - if high_vote != &self.proposal { - return Err(Error::ReproposalBadBlock); - } - } - } - Ok(()) - } -} diff --git a/node/libs/roles/src/validator/messages/leader_proposal.rs b/node/libs/roles/src/validator/messages/leader_proposal.rs new file mode 100644 index 00000000..e05c6668 --- /dev/null +++ b/node/libs/roles/src/validator/messages/leader_proposal.rs @@ -0,0 +1,138 @@ +use super::{ + BlockNumber, CommitQC, CommitQCVerifyError, Genesis, Payload, PayloadHash, TimeoutQC, + TimeoutQCVerifyError, View, +}; + +/// A proposal message from the leader. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LeaderProposal { + /// Payload of the block that the leader is proposing. + /// `None` iff this is a reproposal. + pub proposal_payload: Option, + /// What attests to the validity of this proposal. + pub justification: ProposalJustification, +} + +impl LeaderProposal { + /// View of the message. + pub fn view(&self) -> View { + self.justification.view() + } + + /// Verifies LeaderProposal. + pub fn verify(&self, genesis: &Genesis) -> Result<(), LeaderProposalVerifyError> { + // Check that the justification is valid. + self.justification + .verify(genesis) + .map_err(LeaderProposalVerifyError::Justification) + } +} + +/// Error returned by `LeaderProposal::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum LeaderProposalVerifyError { + /// Invalid Justification. + #[error("Invalid justification: {0:#}")] + Justification(ProposalJustificationVerifyError), +} + +/// Justification for a proposal. This is either a Commit QC or a Timeout QC. +/// The first proposal, for view 0, will always be a timeout. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProposalJustification { + /// This proposal is being proposed after a view where we finalized a block. + /// A commit QC is just a collection of commit votes (with at least + /// QUORUM_WEIGHT) for the previous view. Note that the commit votes MUST + /// be identical. + Commit(CommitQC), + /// This proposal is being proposed after a view where we timed out. + /// A timeout QC is just a collection of timeout votes (with at least + /// QUORUM_WEIGHT) for the previous view. Unlike with the Commit QC, + /// timeout votes don't need to be identical. + /// The first proposal, for view 0, will always be a timeout. + Timeout(TimeoutQC), +} + +impl ProposalJustification { + /// View of the justification. + pub fn view(&self) -> View { + match self { + ProposalJustification::Commit(qc) => qc.view().next(), + ProposalJustification::Timeout(qc) => qc.view.next(), + } + } + + /// Verifies the justification. + pub fn verify(&self, genesis: &Genesis) -> Result<(), ProposalJustificationVerifyError> { + match self { + ProposalJustification::Commit(qc) => qc + .verify(genesis) + .map_err(ProposalJustificationVerifyError::Commit), + ProposalJustification::Timeout(qc) => qc + .verify(genesis) + .map_err(ProposalJustificationVerifyError::Timeout), + } + } + + /// This returns the BlockNumber that is implied by this justification. + /// If the justification requires a block reproposal, it also returns + /// the PayloadHash that must be reproposed. + pub fn get_implied_block(&self, genesis: &Genesis) -> (BlockNumber, Option) { + match self { + ProposalJustification::Commit(qc) => { + // The previous proposal was finalized, so we can propose a new block. + (qc.header().number.next(), None) + } + ProposalJustification::Timeout(qc) => { + // Get the high vote of the timeout QC, if it exists. We check if there are + // timeout votes with at least an added weight of SUBQUORUM_WEIGHT, + // that have a high vote field for the same block. A QC can have + // 0, 1 or 2 such blocks. + // If there's only 1 such block, then we say the QC has a high vote. + // If there are 0 or 2 such blocks, we say the QC has no high vote. + let high_vote = qc.high_vote(genesis); + + // Get the high commit QC of the timeout QC. We compare the high QC field of + // all timeout votes in the QC, and get the highest one, if it exists. + // The high QC always exists, unless no block has been finalized yet in the chain. + let high_qc = qc.high_qc(); + + // If there was a high vote in the timeout QC, and either there was no high QC + // in the timeout QC, or the high vote is for a higher block than the high QC, + // then we need to repropose the high vote. + #[allow(clippy::unnecessary_unwrap)] // using a match would be more verbose + if high_vote.is_some() + && (high_qc.is_none() + || high_vote.unwrap().number > high_qc.unwrap().header().number) + { + // There was some proposal last view that might have been finalized. + // We need to repropose it. + (high_vote.unwrap().number, Some(high_vote.unwrap().payload)) + } else { + // Either the previous proposal was finalized or we know for certain + // that it couldn't have been finalized (because there is no high vote). + // Either way, we can propose a new block. + + // If there is no high QC, then we must be at the start of the chain. + let block_number = match high_qc { + Some(qc) => qc.header().number.next(), + None => genesis.first_block, + }; + + (block_number, None) + } + } + } + } +} + +/// Error returned by `ProposalJustification::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum ProposalJustificationVerifyError { + /// Invalid timeout QC. + #[error("Invalid timeout QC: {0:#}")] + Timeout(TimeoutQCVerifyError), + /// Invalid commit QC. + #[error("Invalid commit QC: {0:#}")] + Commit(CommitQCVerifyError), +} diff --git a/node/libs/roles/src/validator/messages/mod.rs b/node/libs/roles/src/validator/messages/mod.rs index 90189607..5153f832 100644 --- a/node/libs/roles/src/validator/messages/mod.rs +++ b/node/libs/roles/src/validator/messages/mod.rs @@ -1,21 +1,25 @@ //! Messages exchanged between validators. mod block; +mod committee; mod consensus; mod discovery; -mod leader_commit; -mod leader_prepare; +mod genesis; +mod leader_proposal; mod msg; mod replica_commit; -mod replica_prepare; +mod replica_new_view; +mod replica_timeout; #[cfg(test)] mod tests; pub use block::*; +pub use committee::*; pub use consensus::*; pub use discovery::*; -pub use leader_commit::*; -pub use leader_prepare::*; +pub use genesis::*; +pub use leader_proposal::*; pub use msg::*; pub use replica_commit::*; -pub use replica_prepare::*; +pub use replica_new_view::*; +pub use replica_timeout::*; diff --git a/node/libs/roles/src/validator/messages/msg.rs b/node/libs/roles/src/validator/messages/msg.rs index cc5a047e..7fbe6b06 100644 --- a/node/libs/roles/src/validator/messages/msg.rs +++ b/node/libs/roles/src/validator/messages/msg.rs @@ -116,9 +116,9 @@ impl + Clone> Signed { impl> Signed { /// Casts a signed message variant to sub/super variant. /// It is an equivalent of constructing/deconstructing enum values. - pub fn cast>(self) -> Result, BadVariantError> { + pub fn cast>(self) -> Result, BadVariantError> { Ok(Signed { - msg: V2::extract(self.msg.insert())?, + msg: U::extract(self.msg.insert())?, key: self.key, sig: self.sig, }) diff --git a/node/libs/roles/src/validator/messages/replica_commit.rs b/node/libs/roles/src/validator/messages/replica_commit.rs index aaeab7d5..939df5cc 100644 --- a/node/libs/roles/src/validator/messages/replica_commit.rs +++ b/node/libs/roles/src/validator/messages/replica_commit.rs @@ -1,17 +1,7 @@ -use super::{BlockHeader, Genesis, View}; +use super::{BlockHeader, Genesis, Signed, Signers, View}; +use crate::validator; -/// Error returned by `ReplicaCommit::verify()`. -#[derive(thiserror::Error, Debug)] -pub enum ReplicaCommitVerifyError { - /// Invalid view. - #[error("view: {0:#}")] - View(anyhow::Error), - /// Bad block number. - #[error("block number < first block")] - BadBlockNumber, -} - -/// A Commit message from a replica. +/// A commit message from a replica. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ReplicaCommit { /// View of this message. @@ -23,11 +13,186 @@ pub struct ReplicaCommit { impl ReplicaCommit { /// Verifies the message. pub fn verify(&self, genesis: &Genesis) -> Result<(), ReplicaCommitVerifyError> { - use ReplicaCommitVerifyError as Error; - self.view.verify(genesis).map_err(Error::View)?; - if self.proposal.number < genesis.first_block { - return Err(Error::BadBlockNumber); + self.view + .verify(genesis) + .map_err(ReplicaCommitVerifyError::BadView)?; + + Ok(()) + } +} + +/// Error returned by `ReplicaCommit::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum ReplicaCommitVerifyError { + /// Invalid view. + #[error("view: {0:#}")] + BadView(anyhow::Error), +} + +/// A Commit Quorum Certificate. It is an aggregate of signed ReplicaCommit messages. +/// The Commit Quorum Certificate is over identical messages, so we only need one message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CommitQC { + /// The ReplicaCommit message that the QC is for. + pub message: ReplicaCommit, + /// The validators that signed this message. + pub signers: Signers, + /// The aggregate signature of the signed replica messages. + pub signature: validator::AggregateSignature, +} + +impl CommitQC { + /// Header of the certified block. + pub fn header(&self) -> &BlockHeader { + &self.message.proposal + } + + /// View of this QC. + pub fn view(&self) -> &View { + &self.message.view + } + + /// Create a new empty instance for a given `ReplicaCommit` message and a validator set size. + pub fn new(message: ReplicaCommit, genesis: &Genesis) -> Self { + Self { + message, + signers: Signers::new(genesis.validators.len()), + signature: validator::AggregateSignature::default(), } + } + + /// Add a validator's signature. This also verifies the message and the signature before adding. + pub fn add( + &mut self, + msg: &Signed, + genesis: &Genesis, + ) -> Result<(), CommitQCAddError> { + // Check if the signer is in the committee. + let Some(i) = genesis.validators.index(&msg.key) else { + return Err(CommitQCAddError::SignerNotInCommittee { + signer: Box::new(msg.key.clone()), + }); + }; + + // Check if already have a message from the same signer. + if self.signers.0[i] { + return Err(CommitQCAddError::DuplicateSigner { + signer: Box::new(msg.key.clone()), + }); + }; + + // Verify the signature. + msg.verify().map_err(CommitQCAddError::BadSignature)?; + + // Check that the message is consistent with the CommitQC. + if self.message != msg.msg { + return Err(CommitQCAddError::InconsistentMessages); + }; + + // Check that the message itself is valid. + msg.msg + .verify(genesis) + .map_err(CommitQCAddError::InvalidMessage)?; + + // Add the signer to the signers map, and the signature to the aggregate signature. + self.signers.0.set(i, true); + self.signature.add(&msg.sig); + Ok(()) } + + /// Verifies the integrity of the CommitQC. + pub fn verify(&self, genesis: &Genesis) -> Result<(), CommitQCVerifyError> { + // Check that the message is valid. + self.message + .verify(genesis) + .map_err(CommitQCVerifyError::InvalidMessage)?; + + // Check that the signers set has the same size as the validator set. + if self.signers.len() != genesis.validators.len() { + return Err(CommitQCVerifyError::BadSignersSet); + } + + // Verify the signers' weight is enough. + let weight = genesis.validators.weight(&self.signers); + let threshold = genesis.validators.quorum_threshold(); + if weight < threshold { + return Err(CommitQCVerifyError::NotEnoughWeight { + got: weight, + want: threshold, + }); + } + + // Now we can verify the signature. + let messages_and_keys = genesis + .validators + .keys() + .enumerate() + .filter(|(i, _)| self.signers.0[*i]) + .map(|(_, pk)| (self.message.clone(), pk)); + + self.signature + .verify_messages(messages_and_keys) + .map_err(CommitQCVerifyError::BadSignature) + } +} + +impl Ord for CommitQC { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.message.view.number.cmp(&other.message.view.number) + } +} + +impl PartialOrd for CommitQC { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Error returned by `CommitQC::add()`. +#[derive(thiserror::Error, Debug)] +pub enum CommitQCAddError { + /// Signer not present in the committee. + #[error("Signer not in committee: {signer:?}")] + SignerNotInCommittee { + /// Signer of the message. + signer: Box, + }, + /// Message from the same signer already present in QC. + #[error("Message from the same signer already in QC: {signer:?}")] + DuplicateSigner { + /// Signer of the message. + signer: Box, + }, + /// Bad signature. + #[error("Bad signature: {0:#}")] + BadSignature(#[source] anyhow::Error), + /// Inconsistent messages. + #[error("Trying to add signature for a different message")] + InconsistentMessages, + /// Invalid message. + #[error("Invalid message: {0:#}")] + InvalidMessage(ReplicaCommitVerifyError), +} + +/// Error returned by `CommitQC::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum CommitQCVerifyError { + /// Invalid message. + #[error(transparent)] + InvalidMessage(#[from] ReplicaCommitVerifyError), + /// Bad signer set. + #[error("Signers set doesn't match validator set")] + BadSignersSet, + /// Weight not reached. + #[error("Signers have not reached threshold weight: got {got}, want {want}")] + NotEnoughWeight { + /// Got weight. + got: u64, + /// Want weight. + want: u64, + }, + /// Bad signature. + #[error("Bad signature: {0:#}")] + BadSignature(#[source] anyhow::Error), } diff --git a/node/libs/roles/src/validator/messages/replica_new_view.rs b/node/libs/roles/src/validator/messages/replica_new_view.rs new file mode 100644 index 00000000..79b93e25 --- /dev/null +++ b/node/libs/roles/src/validator/messages/replica_new_view.rs @@ -0,0 +1,33 @@ +use super::{Genesis, ProposalJustification, ProposalJustificationVerifyError, View}; + +/// A new view message from a replica. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplicaNewView { + /// What attests to the validity of this view change. + pub justification: ProposalJustification, +} + +impl ReplicaNewView { + /// View of the message. + pub fn view(&self) -> View { + self.justification.view() + } + + /// Verifies ReplicaNewView. + pub fn verify(&self, genesis: &Genesis) -> Result<(), ReplicaNewViewVerifyError> { + // Check that the justification is valid. + self.justification + .verify(genesis) + .map_err(ReplicaNewViewVerifyError::Justification)?; + + Ok(()) + } +} + +/// Error returned by `ReplicaNewView::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum ReplicaNewViewVerifyError { + /// Invalid Justification. + #[error("justification: {0:#}")] + Justification(ProposalJustificationVerifyError), +} diff --git a/node/libs/roles/src/validator/messages/replica_prepare.rs b/node/libs/roles/src/validator/messages/replica_prepare.rs deleted file mode 100644 index 165830f4..00000000 --- a/node/libs/roles/src/validator/messages/replica_prepare.rs +++ /dev/null @@ -1,55 +0,0 @@ -use super::{ - CommitQC, CommitQCVerifyError, Genesis, ReplicaCommit, ReplicaCommitVerifyError, View, -}; - -/// A Prepare message from a replica. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct ReplicaPrepare { - /// View of this message. - pub view: View, - /// The highest block that the replica has committed to. - pub high_vote: Option, - /// The highest CommitQC that the replica has seen. - pub high_qc: Option, -} - -/// Error returned by `ReplicaPrepare::verify()`. -#[derive(thiserror::Error, Debug)] -pub enum ReplicaPrepareVerifyError { - /// View. - #[error("view: {0:#}")] - View(anyhow::Error), - /// FutureHighVoteView. - #[error("high vote from the future")] - HighVoteFutureView, - /// FutureHighQCView. - #[error("high qc from the future")] - HighQCFutureView, - /// HighVote. - #[error("high_vote: {0:#}")] - HighVote(ReplicaCommitVerifyError), - /// HighQC. - #[error("high_qc: {0:#}")] - HighQC(CommitQCVerifyError), -} - -impl ReplicaPrepare { - /// Verifies the message. - pub fn verify(&self, genesis: &Genesis) -> Result<(), ReplicaPrepareVerifyError> { - use ReplicaPrepareVerifyError as Error; - self.view.verify(genesis).map_err(Error::View)?; - if let Some(v) = &self.high_vote { - if self.view.number <= v.view.number { - return Err(Error::HighVoteFutureView); - } - v.verify(genesis).map_err(Error::HighVote)?; - } - if let Some(qc) = &self.high_qc { - if self.view.number <= qc.view().number { - return Err(Error::HighQCFutureView); - } - qc.verify(genesis).map_err(Error::HighQC)?; - } - Ok(()) - } -} diff --git a/node/libs/roles/src/validator/messages/replica_timeout.rs b/node/libs/roles/src/validator/messages/replica_timeout.rs new file mode 100644 index 00000000..0a9edc3f --- /dev/null +++ b/node/libs/roles/src/validator/messages/replica_timeout.rs @@ -0,0 +1,285 @@ +use super::{ + BlockHeader, CommitQC, CommitQCVerifyError, Genesis, ReplicaCommit, ReplicaCommitVerifyError, + Signed, Signers, View, +}; +use crate::validator; +use std::collections::{BTreeMap, HashMap}; + +/// A timeout message from a replica. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ReplicaTimeout { + /// View of this message. + pub view: View, + /// The highest block that the replica has committed to. + pub high_vote: Option, + /// The highest CommitQC that the replica has seen. + pub high_qc: Option, +} + +impl ReplicaTimeout { + /// Verifies the message. + pub fn verify(&self, genesis: &Genesis) -> Result<(), ReplicaTimeoutVerifyError> { + self.view + .verify(genesis) + .map_err(ReplicaTimeoutVerifyError::BadView)?; + + if let Some(v) = &self.high_vote { + v.verify(genesis) + .map_err(ReplicaTimeoutVerifyError::InvalidHighVote)?; + } + + if let Some(qc) = &self.high_qc { + qc.verify(genesis) + .map_err(ReplicaTimeoutVerifyError::InvalidHighQC)?; + } + + Ok(()) + } +} + +/// Error returned by `ReplicaTimeout::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum ReplicaTimeoutVerifyError { + /// View. + #[error("view: {0:#}")] + BadView(anyhow::Error), + /// Invalid High Vote. + #[error("invalid high_vote: {0:#}")] + InvalidHighVote(ReplicaCommitVerifyError), + /// Invalid High QC. + #[error("invalid high_qc: {0:#}")] + InvalidHighQC(CommitQCVerifyError), +} + +/// A quorum certificate of ReplicaTimeout messages. Since not all ReplicaTimeout messages are +/// identical (they have different high blocks and high QCs), we need to keep the ReplicaTimeout +/// messages in a map. We can still aggregate the signatures though. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TimeoutQC { + /// View of this QC. + pub view: View, + /// Map from replica Timeout messages to the validators that signed them. + pub map: BTreeMap, + /// Aggregate signature of the ReplicaTimeout messages. + pub signature: validator::AggregateSignature, +} + +impl TimeoutQC { + /// Create a new empty TimeoutQC for a given view. + pub fn new(view: View) -> Self { + Self { + view, + map: BTreeMap::new(), + signature: validator::AggregateSignature::default(), + } + } + + /// Get the highest block voted and check if there's a subquorum of votes for it. To have a subquorum + /// in this situation, we require n-3*f votes, where f is the maximum number of faulty replicas. + /// Note that it is possible to have 2 subquorums: vote A and vote B, each with >n-3*f weight, in a single + /// TimeoutQC. In such a situation we say that there is no high vote. + pub fn high_vote(&self, genesis: &Genesis) -> Option { + let mut count: HashMap<_, u64> = HashMap::new(); + for (msg, signers) in &self.map { + if let Some(v) = &msg.high_vote { + *count.entry(v.proposal).or_default() += genesis.validators.weight(signers); + } + } + + let min = genesis.validators.subquorum_threshold(); + let mut high_votes: Vec<_> = count.into_iter().filter(|x| x.1 >= min).collect(); + + if high_votes.len() == 1 { + high_votes.pop().map(|x| x.0) + } else { + None + } + } + + /// Get the highest CommitQC. + pub fn high_qc(&self) -> Option<&CommitQC> { + self.map + .keys() + .filter_map(|m| m.high_qc.as_ref()) + .max_by_key(|qc| qc.view().number) + } + + /// Add a validator's signed message. This also verifies the message and the signature before adding. + pub fn add( + &mut self, + msg: &Signed, + genesis: &Genesis, + ) -> Result<(), TimeoutQCAddError> { + // Check if the signer is in the committee. + let Some(i) = genesis.validators.index(&msg.key) else { + return Err(TimeoutQCAddError::SignerNotInCommittee { + signer: Box::new(msg.key.clone()), + }); + }; + + // Check if we already have a message from the same signer. + if self.map.values().any(|s| s.0[i]) { + return Err(TimeoutQCAddError::DuplicateSigner { + signer: Box::new(msg.key.clone()), + }); + }; + + // Verify the signature. + msg.verify().map_err(TimeoutQCAddError::BadSignature)?; + + // Check that the view is consistent with the TimeoutQC. + if msg.msg.view != self.view { + return Err(TimeoutQCAddError::InconsistentViews); + }; + + // Check that the message itself is valid. + msg.msg + .verify(genesis) + .map_err(TimeoutQCAddError::InvalidMessage)?; + + // Add the message plus signer to the map, and the signature to the aggregate signature. + let e = self + .map + .entry(msg.msg.clone()) + .or_insert_with(|| Signers::new(genesis.validators.len())); + e.0.set(i, true); + self.signature.add(&msg.sig); + + Ok(()) + } + + /// Verifies the integrity of the TimeoutQC. + pub fn verify(&self, genesis: &Genesis) -> Result<(), TimeoutQCVerifyError> { + self.view + .verify(genesis) + .map_err(TimeoutQCVerifyError::BadView)?; + + let mut sum = Signers::new(genesis.validators.len()); + + // Check the ReplicaTimeout messages. + for (i, (msg, signers)) in self.map.iter().enumerate() { + if msg.view != self.view { + return Err(TimeoutQCVerifyError::InconsistentView(i)); + } + if signers.len() != sum.len() { + return Err(TimeoutQCVerifyError::WrongSignersLength(i)); + } + if signers.is_empty() { + return Err(TimeoutQCVerifyError::NoSignersAssigned(i)); + } + if !(&sum & signers).is_empty() { + return Err(TimeoutQCVerifyError::OverlappingSignatureSet(i)); + } + msg.verify(genesis) + .map_err(|err| TimeoutQCVerifyError::InvalidMessage(i, err))?; + + sum |= signers; + } + + // Check if the signers' weight is enough. + let weight = genesis.validators.weight(&sum); + let threshold = genesis.validators.quorum_threshold(); + if weight < threshold { + return Err(TimeoutQCVerifyError::NotEnoughWeight { + got: weight, + want: threshold, + }); + } + + // Now we can verify the signature. + let messages_and_keys = self.map.clone().into_iter().flat_map(|(msg, signers)| { + genesis + .validators + .keys() + .enumerate() + .filter(|(i, _)| signers.0[*i]) + .map(|(_, pk)| (msg.clone(), pk)) + .collect::>() + }); + + // TODO(gprusak): This reaggregating is suboptimal. + self.signature + .verify_messages(messages_and_keys) + .map_err(TimeoutQCVerifyError::BadSignature) + } + + /// Calculates the weight of current TimeoutQC signing validators + pub fn weight(&self, committee: &validator::Committee) -> u64 { + self.map + .values() + .map(|signers| committee.weight(signers)) + .sum() + } +} + +impl Ord for TimeoutQC { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.view.number.cmp(&other.view.number) + } +} + +impl PartialOrd for TimeoutQC { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Error returned by `TimeoutQC::add()`. +#[derive(thiserror::Error, Debug)] +pub enum TimeoutQCAddError { + /// Signer not present in the committee. + #[error("Signer not in committee: {signer:?}")] + SignerNotInCommittee { + /// Signer of the message. + signer: Box, + }, + /// Message from the same signer already present in QC. + #[error("Message from the same signer already in QC: {signer:?}")] + DuplicateSigner { + /// Signer of the message. + signer: Box, + }, + /// Bad signature. + #[error("Bad signature: {0:#}")] + BadSignature(#[source] anyhow::Error), + /// Inconsistent views. + #[error("Trying to add a message from a different view")] + InconsistentViews, + /// Invalid message. + #[error("Invalid message: {0:#}")] + InvalidMessage(ReplicaTimeoutVerifyError), +} + +/// Error returned by `TimeoutQC::verify()`. +#[derive(thiserror::Error, Debug)] +pub enum TimeoutQCVerifyError { + /// Bad view. + #[error("Bad view: {0:#}")] + BadView(anyhow::Error), + /// Inconsistent views. + #[error("Message with inconsistent view: number [{0}]")] + InconsistentView(usize), + /// Invalid message. + #[error("Invalid message: number [{0}], {1:#}")] + InvalidMessage(usize, ReplicaTimeoutVerifyError), + /// Wrong signers length. + #[error("Message with wrong signers length: number [{0}]")] + WrongSignersLength(usize), + /// No signers assigned. + #[error("Message with no signers assigned: number [{0}]")] + NoSignersAssigned(usize), + /// Overlapping signature sets. + #[error("Message with overlapping signature set: number [{0}]")] + OverlappingSignatureSet(usize), + /// Weight not reached. + #[error("Signers have not reached threshold weight: got {got}, want {want}")] + NotEnoughWeight { + /// Got weight. + got: u64, + /// Want weight. + want: u64, + }, + /// Bad signature. + #[error("Bad signature: {0:#}")] + BadSignature(#[source] anyhow::Error), +} diff --git a/node/libs/roles/src/validator/messages/tests.rs b/node/libs/roles/src/validator/messages/tests.rs deleted file mode 100644 index c4757280..00000000 --- a/node/libs/roles/src/validator/messages/tests.rs +++ /dev/null @@ -1,359 +0,0 @@ -use crate::{ - attester::{self, WeightedAttester}, - validator::*, -}; -use anyhow::Context as _; -use rand::{prelude::StdRng, Rng, SeedableRng}; -use zksync_concurrency::ctx; -use zksync_consensus_crypto::Text; -use zksync_consensus_utils::enum_util::Variant as _; - -/// Hardcoded secret keys. -fn validator_keys() -> Vec { - [ - "validator:secret:bls12_381:27cb45b1670a1ae8d376a85821d51c7f91ebc6e32788027a84758441aaf0a987", - "validator:secret:bls12_381:20132edc08a529e927f155e710ae7295a2a0d249f1b1f37726894d1d0d8f0d81", - "validator:secret:bls12_381:0946901f0a6650284726763b12de5da0f06df0016c8ec2144cf6b1903f1979a6", - "validator:secret:bls12_381:3143a64c079b2f50545288d7c9b282281e05c97ac043228830a9660ddd63fea3", - "validator:secret:bls12_381:5512f40d33844c1c8107aa630af764005ab6e13f6bf8edb59b4ca3683727e619", - ] - .iter() - .map(|raw| Text::new(raw).decode().unwrap()) - .collect() -} - -fn attester_keys() -> Vec { - [ - "attester:secret:secp256k1:27cb45b1670a1ae8d376a85821d51c7f91ebc6e32788027a84758441aaf0a987", - "attester:secret:secp256k1:20132edc08a529e927f155e710ae7295a2a0d249f1b1f37726894d1d0d8f0d81", - "attester:secret:secp256k1:0946901f0a6650284726763b12de5da0f06df0016c8ec2144cf6b1903f1979a6", - ] - .iter() - .map(|raw| Text::new(raw).decode().unwrap()) - .collect() -} - -/// Hardcoded committee. -fn validator_committee() -> Committee { - Committee::new( - validator_keys() - .iter() - .enumerate() - .map(|(i, key)| WeightedValidator { - key: key.public(), - weight: i as u64 + 10, - }), - ) - .unwrap() -} - -fn attester_committee() -> attester::Committee { - attester::Committee::new( - attester_keys() - .iter() - .enumerate() - .map(|(i, key)| WeightedAttester { - key: key.public(), - weight: i as u64 + 10, - }), - ) - .unwrap() -} - -/// Hardcoded payload. -fn payload() -> Payload { - Payload( - hex::decode("57b79660558f18d56b5196053f64007030a1cb7eeadb5c32d816b9439f77edf5f6bd9d") - .unwrap(), - ) -} - -/// Checks that the order of validators in a committee is stable. -#[test] -fn committee_change_detector() { - let committee = validator_committee(); - let got: Vec = validator_keys() - .iter() - .map(|k| committee.index(&k.public()).unwrap()) - .collect(); - assert_eq!(vec![0, 1, 4, 3, 2], got); -} - -#[test] -fn payload_hash_change_detector() { - let want: PayloadHash = Text::new( - "payload:keccak256:ba8ffff2526cae27a9e8e014749014b08b80e01905c8b769159d02d6579d9b83", - ) - .decode() - .unwrap(); - assert_eq!(want, payload().hash()); -} - -#[test] -fn test_sticky() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let committee = validator_committee(); - let want = committee - .get(rng.gen_range(0..committee.len())) - .unwrap() - .key - .clone(); - let sticky = LeaderSelectionMode::Sticky(want.clone()); - for _ in 0..100 { - assert_eq!(want, committee.view_leader(rng.gen(), &sticky)); - } -} - -#[test] -fn test_rota() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let committee = validator_committee(); - let mut want = Vec::new(); - for _ in 0..3 { - want.push( - committee - .get(rng.gen_range(0..committee.len())) - .unwrap() - .key - .clone(), - ); - } - let rota = LeaderSelectionMode::Rota(want.clone()); - for _ in 0..100 { - let vn: ViewNumber = rng.gen(); - let pk = &want[vn.0 as usize % want.len()]; - assert_eq!(*pk, committee.view_leader(vn, &rota)); - } -} - -/// Hardcoded view numbers. -fn views() -> impl Iterator { - [8394532, 2297897, 9089304, 7203483, 9982111] - .into_iter() - .map(ViewNumber) -} - -/// Checks that leader schedule is stable. -#[test] -fn roundrobin_change_detector() { - let committee = validator_committee(); - let mode = LeaderSelectionMode::RoundRobin; - let got: Vec<_> = views() - .map(|view| { - let got = committee.view_leader(view, &mode); - committee.index(&got).unwrap() - }) - .collect(); - assert_eq!(vec![2, 2, 4, 3, 1], got); -} - -/// Checks that leader schedule is stable. -#[test] -fn weighted_change_detector() { - let committee = validator_committee(); - let mode = LeaderSelectionMode::Weighted; - let got: Vec<_> = views() - .map(|view| { - let got = committee.view_leader(view, &mode); - committee.index(&got).unwrap() - }) - .collect(); - assert_eq!(vec![4, 2, 2, 2, 1], got); -} - -mod version1 { - use super::*; - - /// Hardcoded genesis. - fn genesis_empty_attesters() -> Genesis { - GenesisRaw { - chain_id: ChainId(1337), - fork_number: ForkNumber(402598740274745173), - first_block: BlockNumber(8902834932452), - - protocol_version: ProtocolVersion(1), - validators: validator_committee(), - attesters: None, - leader_selection: LeaderSelectionMode::Weighted, - } - .with_hash() - } - - /// Hardcoded genesis. - fn genesis_with_attesters() -> Genesis { - GenesisRaw { - chain_id: ChainId(1337), - fork_number: ForkNumber(402598740274745173), - first_block: BlockNumber(8902834932452), - - protocol_version: ProtocolVersion(1), - validators: validator_committee(), - attesters: attester_committee().into(), - leader_selection: LeaderSelectionMode::Weighted, - } - .with_hash() - } - - /// Note that genesis is NOT versioned by ProtocolVersion. - /// Even if it was, ALL versions of genesis need to be supported FOREVER, - /// unless we introduce dynamic regenesis. - /// FIXME: This fails with the new attester committee. - #[test] - fn genesis_hash_change_detector_empty_attesters() { - let want: GenesisHash = Text::new( - "genesis_hash:keccak256:13a16cfa758c6716b4c4d40a5fe71023a016c7507b7893c7dc775f4420fc5d61", - ) - .decode() - .unwrap(); - assert_eq!(want, genesis_empty_attesters().hash()); - } - - /// Note that genesis is NOT versioned by ProtocolVersion. - /// Even if it was, ALL versions of genesis need to be supported FOREVER, - /// unless we introduce dynamic regenesis. - /// FIXME: This fails with the new attester committee. - #[test] - fn genesis_hash_change_detector_nonempty_attesters() { - let want: GenesisHash = Text::new( - "genesis_hash:keccak256:47a52a5491873fa4ceb369a334b4c09833a06bd34718fb22e530ab4d70b4daf7", - ) - .decode() - .unwrap(); - assert_eq!(want, genesis_with_attesters().hash()); - } - - #[test] - fn genesis_verify_leader_pubkey_not_in_committee() { - let mut rng = StdRng::seed_from_u64(29483920); - let mut genesis = rng.gen::(); - genesis.leader_selection = LeaderSelectionMode::Sticky(rng.gen()); - let genesis = genesis.with_hash(); - assert!(genesis.verify().is_err()) - } - - /// asserts that msg.hash()==hash and that sig is a - /// valid signature of msg (signed by `keys()[0]`). - #[track_caller] - fn change_detector(msg: Msg, hash: &str, sig: &str) { - let key = validator_keys()[0].clone(); - (|| { - let hash: MsgHash = Text::new(hash).decode()?; - let sig: Signature = Text::new(sig).decode()?; - sig.verify_hash(&hash, &key.public())?; - anyhow::Ok(()) - })() - .with_context(|| format!("\n{:?},\n{:?}", msg.hash(), key.sign_hash(&msg.hash()),)) - .unwrap(); - } - - /// Hardcoded view. - fn view() -> View { - View { - genesis: genesis_empty_attesters().hash(), - number: ViewNumber(9136573498460759103), - } - } - - /// Hardcoded `BlockHeader`. - fn block_header() -> BlockHeader { - BlockHeader { - number: BlockNumber(772839452345), - payload: payload().hash(), - } - } - - /// Hardcoded `ReplicaCommit`. - fn replica_commit() -> ReplicaCommit { - ReplicaCommit { - view: view(), - proposal: block_header(), - } - } - - /// Hardcoded `CommitQC`. - fn commit_qc() -> CommitQC { - let genesis = genesis_empty_attesters(); - let replica_commit = replica_commit(); - let mut x = CommitQC::new(replica_commit.clone(), &genesis); - for k in validator_keys() { - x.add(&k.sign_msg(replica_commit.clone()), &genesis) - .unwrap(); - } - x - } - - /// Hardcoded `LeaderCommit`. - fn leader_commit() -> LeaderCommit { - LeaderCommit { - justification: commit_qc(), - } - } - - /// Hardcoded `ReplicaPrepare` - fn replica_prepare() -> ReplicaPrepare { - ReplicaPrepare { - view: view(), - high_vote: Some(replica_commit()), - high_qc: Some(commit_qc()), - } - } - - /// Hardcoded `PrepareQC`. - fn prepare_qc() -> PrepareQC { - let mut x = PrepareQC::new(view()); - let genesis = genesis_empty_attesters(); - let replica_prepare = replica_prepare(); - for k in validator_keys() { - x.add(&k.sign_msg(replica_prepare.clone()), &genesis) - .unwrap(); - } - x - } - - /// Hardcoded `LeaderPrepare`. - fn leader_prepare() -> LeaderPrepare { - LeaderPrepare { - proposal: block_header(), - proposal_payload: Some(payload()), - justification: prepare_qc(), - } - } - - #[test] - fn replica_commit_change_detector() { - change_detector( - replica_commit().insert(), - "validator_msg:keccak256:2ec798684e539d417fac1caba74ed1a27a033bc18058ba0a4632f6bb0ae4fe1c", - "validator:signature:bls12_381:8de9ad850d78eb4f918c8c3a02310be49fc9ac35f2b1fdd6489293db1d5128f0d4c8389674e6bc2eee4c6e16f58e0b51", - ); - } - - #[test] - fn leader_commit_change_detector() { - change_detector( - leader_commit().insert(), - "validator_msg:keccak256:53b8d7dc77a5ba8b81cd7b46ddc19c224ef46245c26cb7ae12239acc1bf86eda", - "validator:signature:bls12_381:81a154a93a8b607031319915728be97c03c3014a4746050f7a32cde98cabe4fbd2b6d6b79400601a71f50350842d1d64", - ); - } - - #[test] - fn replica_prepare_change_detector() { - change_detector( - replica_prepare().insert(), - "validator_msg:keccak256:700cf26d50f463cfa908f914d1febb1cbd00ee9d3a691b644f49146ed3e6ac40", - "validator:signature:bls12_381:a7cbdf9b8d13ebc39f4a13d654ec30acccd247d46fc6121eb1220256cfc212b418aac85400176e8797d8eb91aa70ae78", - ); - } - - #[test] - fn leader_prepare_change_detector() { - change_detector( - leader_prepare().insert(), - "validator_msg:keccak256:aaaaa6b7b232ef5b7c797953ce2a042c024137d7b8f449a1ad8a535730bc269b", - "validator:signature:bls12_381:a1926f460fa63470544cc9213e6378f45d75dff3055924766a81ff696a6a6e85ee583707911bb7fef4d1f74b7b28132f", - ); - } -} diff --git a/node/libs/roles/src/validator/messages/tests/block.rs b/node/libs/roles/src/validator/messages/tests/block.rs new file mode 100644 index 00000000..e3a3ea52 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/block.rs @@ -0,0 +1,71 @@ +use super::*; +use assert_matches::assert_matches; +use rand::Rng; +use validator::testonly::Setup; +use zksync_concurrency::ctx; +use zksync_consensus_crypto::{keccak256::Keccak256, Text}; + +#[test] +fn payload_hash_change_detector() { + let want: PayloadHash = Text::new( + "payload:keccak256:ba8ffff2526cae27a9e8e014749014b08b80e01905c8b769159d02d6579d9b83", + ) + .decode() + .unwrap(); + assert_eq!(want, payload().hash()); +} + +#[test] +fn test_payload_hash() { + let data = vec![1, 2, 3, 4]; + let payload = Payload(data.clone()); + let hash = payload.hash(); + assert_eq!(hash.0, Keccak256::new(&data)); +} + +#[test] +fn test_block_number_next() { + let block_number = BlockNumber(5); + assert_eq!(block_number.next(), BlockNumber(6)); +} + +#[test] +fn test_block_number_prev() { + let block_number = BlockNumber(5); + assert_eq!(block_number.prev(), Some(BlockNumber(4))); + + let block_number_zero = BlockNumber(0); + assert_eq!(block_number_zero.prev(), None); +} + +#[test] +fn test_block_number_add() { + let block_number = BlockNumber(5); + assert_eq!(block_number + 3, BlockNumber(8)); +} + +#[test] +fn test_final_block_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let setup = Setup::new(rng, 2); + + let payload: Payload = rng.gen(); + let view_number = rng.gen(); + let commit_qc = setup.make_commit_qc_with_payload(&payload, view_number); + let mut final_block = FinalBlock::new(payload.clone(), commit_qc.clone()); + + assert!(final_block.verify(&setup.genesis).is_ok()); + + final_block.payload = rng.gen(); + assert_matches!( + final_block.verify(&setup.genesis), + Err(BlockValidationError::HashMismatch { .. }) + ); + + final_block.justification.message.proposal.payload = final_block.payload.hash(); + assert_matches!( + final_block.verify(&setup.genesis), + Err(BlockValidationError::Justification(_)) + ); +} diff --git a/node/libs/roles/src/validator/messages/tests/committee.rs b/node/libs/roles/src/validator/messages/tests/committee.rs new file mode 100644 index 00000000..0b9968f9 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/committee.rs @@ -0,0 +1,198 @@ +use super::*; +use rand::Rng; +use zksync_concurrency::ctx; + +/// Checks that the order of validators in a committee is stable. +#[test] +fn test_committee_order_change_detector() { + let committee = validator_committee(); + let got: Vec = validator_keys() + .iter() + .map(|k| committee.index(&k.public()).unwrap()) + .collect(); + assert_eq!(vec![0, 1, 4, 3, 2], got); +} + +fn create_validator(weight: u64) -> WeightedValidator { + WeightedValidator { + key: validator::SecretKey::generate().public(), + weight, + } +} + +#[test] +fn test_committee_new() { + let validators = vec![create_validator(10), create_validator(20)]; + let committee = Committee::new(validators).unwrap(); + assert_eq!(committee.len(), 2); + assert_eq!(committee.total_weight(), 30); +} + +#[test] +fn test_committee_new_duplicate_validator() { + let mut validators = vec![create_validator(10), create_validator(20)]; + validators[1].key = validators[0].key.clone(); + let result = Committee::new(validators); + assert!(result.is_err()); +} + +#[test] +fn test_committee_new_zero_weight() { + let validators = vec![create_validator(10), create_validator(0)]; + let result = Committee::new(validators); + assert!(result.is_err()); +} + +#[test] +fn test_committee_weights_overflow_check() { + let validators: Vec = [u64::MAX / 5; 6] + .iter() + .map(|w| create_validator(*w)) + .collect(); + let result = Committee::new(validators); + assert!(result.is_err()); +} + +#[test] +fn test_committee_new_empty() { + let validators = vec![]; + let result = Committee::new(validators); + assert!(result.is_err()); +} + +#[test] +fn test_committee_contains() { + let validators = vec![create_validator(10), create_validator(20)]; + let committee = Committee::new(validators.clone()).unwrap(); + assert!(committee.contains(&validators[0].key)); + assert!(!committee.contains(&validator::SecretKey::generate().public())); +} + +#[test] +fn test_committee_get() { + let validators = validator_keys() + .into_iter() + .map(|x| x.public()) + .collect::>(); + let committee = validator_committee(); + assert_eq!(committee.get(0).unwrap().key, validators[0]); + assert_eq!(committee.get(1).unwrap().key, validators[1]); + assert_eq!(committee.get(2).unwrap().key, validators[4]); + assert_eq!(committee.get(3).unwrap().key, validators[3]); + assert_eq!(committee.get(4).unwrap().key, validators[2]); + assert!(committee.get(5).is_none()); +} + +#[test] +fn test_committee_index() { + let validators = validator_keys() + .into_iter() + .map(|x| x.public()) + .collect::>(); + let committee = validator_committee(); + assert_eq!(committee.index(&validators[0]), Some(0)); + assert_eq!(committee.index(&validators[1]), Some(1)); + assert_eq!(committee.index(&validators[4]), Some(2)); + assert_eq!(committee.index(&validators[3]), Some(3)); + assert_eq!(committee.index(&validators[2]), Some(4)); + assert_eq!( + committee.index(&validator::SecretKey::generate().public()), + None + ); +} + +#[test] +fn test_committee_view_leader_round_robin() { + let committee = validator_committee(); + let mode = LeaderSelectionMode::RoundRobin; + let got: Vec<_> = views() + .map(|view| { + let got = committee.view_leader(view, &mode); + committee.index(&got).unwrap() + }) + .collect(); + assert_eq!(vec![2, 3, 4, 4, 1], got); +} + +#[test] +fn test_committee_view_leader_weighted() { + let committee = validator_committee(); + let mode = LeaderSelectionMode::Weighted; + let got: Vec<_> = views() + .map(|view| { + let got = committee.view_leader(view, &mode); + committee.index(&got).unwrap() + }) + .collect(); + assert_eq!(vec![2, 3, 2, 1, 3], got); +} + +#[test] +fn test_committee_view_leader_sticky() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let committee = validator_committee(); + let want = committee + .get(rng.gen_range(0..committee.len())) + .unwrap() + .key + .clone(); + let sticky = LeaderSelectionMode::Sticky(want.clone()); + for _ in 0..100 { + assert_eq!(want, committee.view_leader(rng.gen(), &sticky)); + } +} + +#[test] +fn test_committee_view_leader_rota() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let committee = validator_committee(); + let mut want = Vec::new(); + for _ in 0..3 { + want.push( + committee + .get(rng.gen_range(0..committee.len())) + .unwrap() + .key + .clone(), + ); + } + let rota = LeaderSelectionMode::Rota(want.clone()); + for _ in 0..100 { + let vn: ViewNumber = rng.gen(); + let pk = &want[vn.0 as usize % want.len()]; + assert_eq!(*pk, committee.view_leader(vn, &rota)); + } +} + +#[test] +fn test_committee_quorum_threshold() { + let validators = vec![create_validator(10), create_validator(20)]; + let committee = Committee::new(validators).unwrap(); + assert_eq!(committee.quorum_threshold(), 25); // 30 - (30 - 1) / 5 +} + +#[test] +fn test_committee_subquorum_threshold() { + let validators = vec![create_validator(10), create_validator(20)]; + let committee = Committee::new(validators).unwrap(); + assert_eq!(committee.subquorum_threshold(), 15); // 30 - 3 * (30 - 1) / 5 +} + +#[test] +fn test_committee_max_faulty_weight() { + let validators = vec![create_validator(10), create_validator(20)]; + let committee = Committee::new(validators).unwrap(); + assert_eq!(committee.max_faulty_weight(), 5); // (30 - 1) / 5 +} + +#[test] +fn test_committee_weight() { + let committee = validator_committee(); + let mut signers = Signers::new(5); + signers.0.set(1, true); + signers.0.set(2, true); + signers.0.set(4, true); + assert_eq!(committee.weight(&signers), 37); +} diff --git a/node/libs/roles/src/validator/messages/tests/consensus.rs b/node/libs/roles/src/validator/messages/tests/consensus.rs new file mode 100644 index 00000000..632f7508 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/consensus.rs @@ -0,0 +1,92 @@ +use super::*; + +#[test] +fn test_view_next() { + let view = View { + genesis: GenesisHash::default(), + number: ViewNumber(1), + }; + let next_view = view.next(); + assert_eq!(next_view.number, ViewNumber(2)); +} + +#[test] +fn test_view_prev() { + let view = View { + genesis: GenesisHash::default(), + number: ViewNumber(1), + }; + let prev_view = view.prev(); + assert_eq!(prev_view.unwrap().number, ViewNumber(0)); + let view = View { + genesis: GenesisHash::default(), + number: ViewNumber(0), + }; + let prev_view = view.prev(); + assert!(prev_view.is_none()); +} + +#[test] +fn test_view_verify() { + let genesis = genesis_with_attesters(); + let view = View { + genesis: genesis.hash(), + number: ViewNumber(1), + }; + assert!(view.verify(&genesis).is_ok()); + assert!(view.verify(&genesis_empty_attesters()).is_err()); + let view = View { + genesis: GenesisHash::default(), + number: ViewNumber(1), + }; + assert!(view.verify(&genesis).is_err()); +} + +#[test] +fn test_signers_new() { + let signers = Signers::new(10); + assert_eq!(signers.len(), 10); + assert!(signers.is_empty()); +} + +#[test] +fn test_signers_count() { + let mut signers = Signers::new(10); + signers.0.set(0, true); + signers.0.set(1, true); + assert_eq!(signers.count(), 2); +} + +#[test] +fn test_signers_empty() { + let mut signers = Signers::new(10); + assert!(signers.is_empty()); + signers.0.set(1, true); + assert!(!signers.is_empty()); + signers.0.set(1, false); + assert!(signers.is_empty()); +} + +#[test] +fn test_signers_bitor_assign() { + let mut signers1 = Signers::new(10); + let mut signers2 = Signers::new(10); + signers1.0.set(0, true); + signers1.0.set(3, true); + signers2.0.set(1, true); + signers2.0.set(3, true); + signers1 |= &signers2; + assert_eq!(signers1.count(), 3); +} + +#[test] +fn test_signers_bitand_assign() { + let mut signers1 = Signers::new(10); + let mut signers2 = Signers::new(10); + signers1.0.set(0, true); + signers1.0.set(3, true); + signers2.0.set(1, true); + signers2.0.set(3, true); + signers1 &= &signers2; + assert_eq!(signers1.count(), 1); +} diff --git a/node/libs/roles/src/validator/messages/tests/genesis.rs b/node/libs/roles/src/validator/messages/tests/genesis.rs new file mode 100644 index 00000000..cdeac273 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/genesis.rs @@ -0,0 +1,30 @@ +use super::*; +use rand::{prelude::StdRng, Rng, SeedableRng}; +use validator::testonly::Setup; +use zksync_concurrency::ctx; +use zksync_protobuf::ProtoFmt as _; + +#[test] +fn genesis_verify_leader_pubkey_not_in_committee() { + let mut rng = StdRng::seed_from_u64(29483920); + let mut genesis = rng.gen::(); + genesis.leader_selection = LeaderSelectionMode::Sticky(rng.gen()); + let genesis = genesis.with_hash(); + assert!(genesis.verify().is_err()) +} + +#[test] +fn test_genesis_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + let genesis = Setup::new(rng, 1).genesis.clone(); + assert!(genesis.verify().is_ok()); + assert!(Genesis::read(&genesis.build()).is_ok()); + + let mut genesis = (*genesis).clone(); + genesis.leader_selection = LeaderSelectionMode::Sticky(rng.gen()); + let genesis = genesis.with_hash(); + assert!(genesis.verify().is_err()); + assert!(Genesis::read(&genesis.build()).is_err()) +} diff --git a/node/libs/roles/src/validator/messages/tests/leader_proposal.rs b/node/libs/roles/src/validator/messages/tests/leader_proposal.rs new file mode 100644 index 00000000..54b99946 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/leader_proposal.rs @@ -0,0 +1,76 @@ +use super::*; +use assert_matches::assert_matches; +use zksync_concurrency::ctx; + +#[test] +fn test_leader_proposal_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + // This will create equally weighted validators + let mut setup = Setup::new(rng, 6); + setup.push_blocks(rng, 3); + + // Valid proposal + let payload: Payload = rng.gen(); + let commit_qc = match setup.blocks.last().unwrap() { + Block::Final(block) => block.justification.clone(), + _ => unreachable!(), + }; + let justification = ProposalJustification::Commit(commit_qc); + let proposal = LeaderProposal { + proposal_payload: Some(payload.clone()), + justification, + }; + + assert!(proposal.verify(&setup.genesis).is_ok()); + + // Invalid justification + let mut wrong_proposal = proposal.clone(); + wrong_proposal.justification = ProposalJustification::Timeout(rng.gen()); + + assert_matches!( + wrong_proposal.verify(&setup.genesis), + Err(LeaderProposalVerifyError::Justification(_)) + ); +} + +#[test] +fn test_justification_get_implied_block() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let mut setup = Setup::new(rng, 6); + setup.push_blocks(rng, 3); + let payload: Payload = rng.gen(); + + // Justification with a commit QC + let commit_qc = match setup.blocks.last().unwrap() { + Block::Final(block) => block.justification.clone(), + _ => unreachable!(), + }; + let justification = ProposalJustification::Commit(commit_qc); + let proposal = LeaderProposal { + proposal_payload: Some(payload.clone()), + justification, + }; + + let (implied_block_number, implied_payload) = + proposal.justification.get_implied_block(&setup.genesis); + + assert_eq!(implied_block_number, setup.next()); + assert!(implied_payload.is_none()); + + // Justification with a timeout QC + let timeout_qc = setup.make_timeout_qc(rng, ViewNumber(7), Some(&payload)); + let justification = ProposalJustification::Timeout(timeout_qc); + let proposal = LeaderProposal { + proposal_payload: None, + justification, + }; + + let (implied_block_number, implied_payload) = + proposal.justification.get_implied_block(&setup.genesis); + + assert_eq!(implied_block_number, setup.next()); + assert_eq!(implied_payload, Some(payload.hash())); +} diff --git a/node/libs/roles/src/validator/messages/tests/mod.rs b/node/libs/roles/src/validator/messages/tests/mod.rs new file mode 100644 index 00000000..798f74ed --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/mod.rs @@ -0,0 +1,191 @@ +use super::*; +use crate::{ + attester::{self, WeightedAttester}, + validator::{self, testonly::Setup}, +}; +use rand::Rng; +use zksync_consensus_crypto::Text; + +mod block; +mod committee; +mod consensus; +mod genesis; +mod leader_proposal; +mod replica_commit; +mod replica_timeout; +mod versions; + +/// Hardcoded view. +fn view() -> View { + View { + genesis: genesis_empty_attesters().hash(), + number: ViewNumber(9136), + } +} + +/// Hardcoded view numbers. +fn views() -> impl Iterator { + [2297, 7203, 8394, 9089, 99821].into_iter().map(ViewNumber) +} + +/// Hardcoded payload. +fn payload() -> Payload { + Payload( + hex::decode("57b79660558f18d56b5196053f64007030a1cb7eeadb5c32d816b9439f77edf5f6bd9d") + .unwrap(), + ) +} + +/// Hardcoded `BlockHeader`. +fn block_header() -> BlockHeader { + BlockHeader { + number: BlockNumber(7728), + payload: payload().hash(), + } +} + +/// Hardcoded validator secret keys. +fn validator_keys() -> Vec { + [ + "validator:secret:bls12_381:27cb45b1670a1ae8d376a85821d51c7f91ebc6e32788027a84758441aaf0a987", + "validator:secret:bls12_381:20132edc08a529e927f155e710ae7295a2a0d249f1b1f37726894d1d0d8f0d81", + "validator:secret:bls12_381:0946901f0a6650284726763b12de5da0f06df0016c8ec2144cf6b1903f1979a6", + "validator:secret:bls12_381:3143a64c079b2f50545288d7c9b282281e05c97ac043228830a9660ddd63fea3", + "validator:secret:bls12_381:5512f40d33844c1c8107aa630af764005ab6e13f6bf8edb59b4ca3683727e619", + ] + .iter() + .map(|raw| Text::new(raw).decode().unwrap()) + .collect() +} + +/// Hardcoded attester secret keys. +fn attester_keys() -> Vec { + [ + "attester:secret:secp256k1:27cb45b1670a1ae8d376a85821d51c7f91ebc6e32788027a84758441aaf0a987", + "attester:secret:secp256k1:20132edc08a529e927f155e710ae7295a2a0d249f1b1f37726894d1d0d8f0d81", + "attester:secret:secp256k1:0946901f0a6650284726763b12de5da0f06df0016c8ec2144cf6b1903f1979a6", + ] + .iter() + .map(|raw| Text::new(raw).decode().unwrap()) + .collect() +} + +/// Hardcoded validator committee. +fn validator_committee() -> Committee { + Committee::new( + validator_keys() + .iter() + .enumerate() + .map(|(i, key)| WeightedValidator { + key: key.public(), + weight: i as u64 + 10, + }), + ) + .unwrap() +} + +/// Hardcoded attester committee. +fn attester_committee() -> attester::Committee { + attester::Committee::new( + attester_keys() + .iter() + .enumerate() + .map(|(i, key)| WeightedAttester { + key: key.public(), + weight: i as u64 + 10, + }), + ) + .unwrap() +} + +/// Hardcoded genesis with no attesters. +fn genesis_empty_attesters() -> Genesis { + GenesisRaw { + chain_id: ChainId(1337), + fork_number: ForkNumber(42), + first_block: BlockNumber(2834), + + protocol_version: ProtocolVersion(1), + validators: validator_committee(), + attesters: None, + leader_selection: LeaderSelectionMode::Weighted, + } + .with_hash() +} + +/// Hardcoded genesis with attesters. +fn genesis_with_attesters() -> Genesis { + GenesisRaw { + chain_id: ChainId(1337), + fork_number: ForkNumber(42), + first_block: BlockNumber(2834), + + protocol_version: ProtocolVersion(1), + validators: validator_committee(), + attesters: attester_committee().into(), + leader_selection: LeaderSelectionMode::Weighted, + } + .with_hash() +} + +/// Hardcoded `LeaderProposal`. +fn leader_proposal() -> LeaderProposal { + LeaderProposal { + proposal_payload: Some(payload()), + justification: ProposalJustification::Timeout(timeout_qc()), + } +} + +/// Hardcoded `ReplicaCommit`. +fn replica_commit() -> ReplicaCommit { + ReplicaCommit { + view: view(), + proposal: block_header(), + } +} + +/// Hardcoded `CommitQC`. +fn commit_qc() -> CommitQC { + let genesis = genesis_empty_attesters(); + let replica_commit = replica_commit(); + let mut x = CommitQC::new(replica_commit.clone(), &genesis); + for k in validator_keys() { + x.add(&k.sign_msg(replica_commit.clone()), &genesis) + .unwrap(); + } + x +} + +/// Hardcoded `ReplicaTimeout` +fn replica_timeout() -> ReplicaTimeout { + ReplicaTimeout { + view: View { + genesis: genesis_empty_attesters().hash(), + number: ViewNumber(9169), + }, + high_vote: Some(replica_commit()), + high_qc: Some(commit_qc()), + } +} + +/// Hardcoded `TimeoutQC`. +fn timeout_qc() -> TimeoutQC { + let mut x = TimeoutQC::new(View { + genesis: genesis_empty_attesters().hash(), + number: ViewNumber(9169), + }); + let genesis = genesis_empty_attesters(); + let replica_timeout = replica_timeout(); + for k in validator_keys() { + x.add(&k.sign_msg(replica_timeout.clone()), &genesis) + .unwrap(); + } + x +} + +/// Hardcoded `ReplicaNewView`. +fn replica_new_view() -> ReplicaNewView { + ReplicaNewView { + justification: ProposalJustification::Commit(commit_qc()), + } +} diff --git a/node/libs/roles/src/validator/messages/tests/replica_commit.rs b/node/libs/roles/src/validator/messages/tests/replica_commit.rs new file mode 100644 index 00000000..d7b9d349 --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/replica_commit.rs @@ -0,0 +1,139 @@ +use super::*; +use assert_matches::assert_matches; +use zksync_concurrency::ctx; + +#[test] +fn test_replica_commit_verify() { + let genesis = genesis_empty_attesters(); + let commit = replica_commit(); + assert!(commit.verify(&genesis).is_ok()); + + // Wrong view + let wrong_genesis = genesis_with_attesters().clone(); + assert_matches!( + commit.verify(&wrong_genesis), + Err(ReplicaCommitVerifyError::BadView(_)) + ); +} + +#[test] +fn test_commit_qc_add() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let setup = Setup::new(rng, 2); + let view = rng.gen(); + let mut qc = CommitQC::new(setup.make_replica_commit(rng, view), &setup.genesis); + let msg = qc.message.clone(); + + // Add the first signature + assert_eq!(qc.signers.count(), 0); + assert!(qc + .add( + &setup.validator_keys[0].sign_msg(msg.clone()), + &setup.genesis + ) + .is_ok()); + assert_eq!(qc.signers.count(), 1); + + // Try to add a signature from a signer not in committee + assert_matches!( + qc.add( + &rng.gen::().sign_msg(msg.clone()), + &setup.genesis + ), + Err(CommitQCAddError::SignerNotInCommittee { .. }) + ); + + // Try to add a signature from the same validator + assert_matches!( + qc.add( + &setup.validator_keys[0].sign_msg(msg.clone()), + &setup.genesis + ), + Err(CommitQCAddError::DuplicateSigner { .. }) + ); + + // Try to add an invalid signature + assert_matches!( + qc.add( + &Signed { + msg: msg.clone(), + key: setup.validator_keys[1].public(), + sig: rng.gen() + }, + &setup.genesis + ), + Err(CommitQCAddError::BadSignature(_)) + ); + + // Try to add a signature for a different message + let mut msg1 = msg.clone(); + msg1.view.number = view.next(); + assert_matches!( + qc.add(&setup.validator_keys[1].sign_msg(msg1), &setup.genesis), + Err(CommitQCAddError::InconsistentMessages) + ); + + // Try to add an invalid message + let mut wrong_genesis = setup.genesis.clone().0; + wrong_genesis.chain_id = rng.gen(); + assert_matches!( + qc.add( + &setup.validator_keys[1].sign_msg(msg.clone()), + &wrong_genesis.with_hash() + ), + Err(CommitQCAddError::InvalidMessage(_)) + ); + + // Add same message signed by another validator. + assert_matches!( + qc.add(&setup.validator_keys[1].sign_msg(msg), &setup.genesis), + Ok(()) + ); + assert_eq!(qc.signers.count(), 2); +} + +#[test] +fn test_commit_qc_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let setup = Setup::new(rng, 6); + let view = rng.gen(); + let qc = setup.make_commit_qc(rng, view); + + // Verify the QC + assert!(qc.verify(&setup.genesis).is_ok()); + + // QC with bad message + let mut qc1 = qc.clone(); + qc1.message.view.genesis = rng.gen(); + assert_matches!( + qc1.verify(&setup.genesis), + Err(CommitQCVerifyError::InvalidMessage(_)) + ); + + // QC with too many signers + let mut qc2 = qc.clone(); + qc2.signers = Signers::new(setup.genesis.validators.len() + 1); + assert_matches!( + qc2.verify(&setup.genesis), + Err(CommitQCVerifyError::BadSignersSet) + ); + + // QC with not enough weight + let mut qc3 = qc.clone(); + qc3.signers.0.set(0, false); + qc3.signers.0.set(4, false); + assert_matches!( + qc3.verify(&setup.genesis), + Err(CommitQCVerifyError::NotEnoughWeight { .. }) + ); + + // QC with bad signature + let mut qc4 = qc.clone(); + qc4.signature = rng.gen(); + assert_matches!( + qc4.verify(&setup.genesis), + Err(CommitQCVerifyError::BadSignature(_)) + ); +} diff --git a/node/libs/roles/src/validator/messages/tests/replica_timeout.rs b/node/libs/roles/src/validator/messages/tests/replica_timeout.rs new file mode 100644 index 00000000..09e5c49c --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/replica_timeout.rs @@ -0,0 +1,332 @@ +use super::*; +use assert_matches::assert_matches; +use zksync_concurrency::ctx; + +#[test] +fn test_replica_timeout_verify() { + let genesis = genesis_empty_attesters(); + let timeout = replica_timeout(); + assert!(timeout.verify(&genesis).is_ok()); + + // Wrong view + let wrong_genesis = genesis_with_attesters().clone(); + assert_matches!( + timeout.verify(&wrong_genesis), + Err(ReplicaTimeoutVerifyError::BadView(_)) + ); + + // Invalid high vote + let mut timeout = replica_timeout(); + timeout.high_vote.as_mut().unwrap().view.genesis = genesis_with_attesters().hash(); + assert_matches!( + timeout.verify(&genesis), + Err(ReplicaTimeoutVerifyError::InvalidHighVote(_)) + ); + + // Invalid high QC + let mut timeout = replica_timeout(); + timeout.high_qc.as_mut().unwrap().message.view.genesis = genesis_with_attesters().hash(); + assert_matches!( + timeout.verify(&genesis), + Err(ReplicaTimeoutVerifyError::InvalidHighQC(_)) + ); +} + +#[test] +fn test_timeout_qc_high_vote() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + + // This will create equally weighted validators + let setup = Setup::new(rng, 6); + + let view_num: ViewNumber = rng.gen(); + let msg_a = setup.make_replica_timeout(rng, view_num); + let msg_b = setup.make_replica_timeout(rng, view_num); + let msg_c = setup.make_replica_timeout(rng, view_num); + + // Case with 1 subquorum. + let mut qc = TimeoutQC::new(msg_a.view); + + for key in &setup.validator_keys { + qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) + .unwrap(); + } + + assert!(qc.high_vote(&setup.genesis).is_some()); + + // Case with 2 subquorums. + let mut qc = TimeoutQC::new(msg_a.view); + + for key in &setup.validator_keys[0..3] { + qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) + .unwrap(); + } + + for key in &setup.validator_keys[3..6] { + qc.add(&key.sign_msg(msg_b.clone()), &setup.genesis) + .unwrap(); + } + + assert!(qc.high_vote(&setup.genesis).is_none()); + + // Case with no subquorums. + let mut qc = TimeoutQC::new(msg_a.view); + + for key in &setup.validator_keys[0..2] { + qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) + .unwrap(); + } + + for key in &setup.validator_keys[2..4] { + qc.add(&key.sign_msg(msg_b.clone()), &setup.genesis) + .unwrap(); + } + + for key in &setup.validator_keys[4..6] { + qc.add(&key.sign_msg(msg_c.clone()), &setup.genesis) + .unwrap(); + } + + assert!(qc.high_vote(&setup.genesis).is_none()); +} + +#[test] +fn test_timeout_qc_high_qc() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let setup = Setup::new(rng, 3); + let view = View { + genesis: setup.genesis.hash(), + number: ViewNumber(100), + }; + let mut qc = TimeoutQC::new(view); + + // No high QC + assert!(qc.high_qc().is_none()); + + // Add signatures with different high QC views + for i in 0..3 { + let high_vote_view = view.number; + let high_qc_view = ViewNumber(view.number.0 - i as u64); + let msg = ReplicaTimeout { + view: setup.make_view(view.number), + high_vote: Some(setup.make_replica_commit(rng, high_vote_view)), + high_qc: Some(setup.make_commit_qc(rng, high_qc_view)), + }; + qc.add( + &setup.validator_keys[i].sign_msg(msg.clone()), + &setup.genesis, + ) + .unwrap(); + } + + assert_eq!(qc.high_qc().unwrap().message.view.number, view.number); +} + +#[test] +fn test_timeout_qc_add() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let setup = Setup::new(rng, 3); + let view = rng.gen(); + let msg = setup.make_replica_timeout(rng, view); + let mut qc = TimeoutQC::new(msg.view); + + // Add the first signature + assert!(qc.map.is_empty()); + assert!(qc + .add( + &setup.validator_keys[0].sign_msg(msg.clone()), + &setup.genesis + ) + .is_ok()); + assert_eq!(qc.map.len(), 1); + assert_eq!(qc.map.values().next().unwrap().count(), 1); + + // Try to add a message from a signer not in committee + assert_matches!( + qc.add( + &rng.gen::().sign_msg(msg.clone()), + &setup.genesis + ), + Err(TimeoutQCAddError::SignerNotInCommittee { .. }) + ); + + // Try to add the same message already added by same validator + assert_matches!( + qc.add( + &setup.validator_keys[0].sign_msg(msg.clone()), + &setup.genesis + ), + Err(TimeoutQCAddError::DuplicateSigner { .. }) + ); + + // Try to add an invalid signature + assert_matches!( + qc.add( + &Signed { + msg: msg.clone(), + key: setup.validator_keys[1].public(), + sig: rng.gen() + }, + &setup.genesis + ), + Err(TimeoutQCAddError::BadSignature(_)) + ); + + // Try to add a message with a different view + let mut msg1 = msg.clone(); + msg1.view.number = view.next(); + assert_matches!( + qc.add(&setup.validator_keys[1].sign_msg(msg1), &setup.genesis), + Err(TimeoutQCAddError::InconsistentViews) + ); + + // Try to add an invalid message + let mut wrong_genesis = setup.genesis.clone().0; + wrong_genesis.chain_id = rng.gen(); + assert_matches!( + qc.add( + &setup.validator_keys[1].sign_msg(msg.clone()), + &wrong_genesis.with_hash() + ), + Err(TimeoutQCAddError::InvalidMessage(_)) + ); + + // Add same message signed by another validator. + assert!(qc + .add( + &setup.validator_keys[1].sign_msg(msg.clone()), + &setup.genesis + ) + .is_ok()); + assert_eq!(qc.map.len(), 1); + assert_eq!(qc.map.values().next().unwrap().count(), 2); + + // Add a different message signed by another validator. + let msg2 = setup.make_replica_timeout(rng, view); + assert!(qc + .add( + &setup.validator_keys[2].sign_msg(msg2.clone()), + &setup.genesis + ) + .is_ok()); + assert_eq!(qc.map.len(), 2); + assert_eq!(qc.map.values().next().unwrap().count(), 2); + assert_eq!(qc.map.values().last().unwrap().count(), 1); +} + +#[test] +fn test_timeout_qc_verify() { + let ctx = ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + let mut setup = Setup::new(rng, 6); + setup.push_blocks(rng, 2); + let view = rng.gen(); + let qc = setup.make_timeout_qc(rng, view, None); + + // Verify the QC + assert!(qc.verify(&setup.genesis).is_ok()); + + // QC with bad view + let mut qc1 = qc.clone(); + qc1.view = rng.gen(); + assert_matches!( + qc1.verify(&setup.genesis), + Err(TimeoutQCVerifyError::BadView(_)) + ); + + // QC with message with inconsistent view + let mut qc2 = qc.clone(); + qc2.map.insert( + ReplicaTimeout { + view: qc2.view.next(), + high_vote: None, + high_qc: None, + }, + Signers::new(setup.genesis.validators.len()), + ); + assert_matches!( + qc2.verify(&setup.genesis), + Err(TimeoutQCVerifyError::InconsistentView(_)) + ); + + // QC with message with wrong signer length + let mut qc3 = qc.clone(); + qc3.map.insert( + ReplicaTimeout { + view: qc3.view, + high_vote: None, + high_qc: None, + }, + Signers::new(setup.genesis.validators.len() + 1), + ); + assert_matches!( + qc3.verify(&setup.genesis), + Err(TimeoutQCVerifyError::WrongSignersLength(_)) + ); + + // QC with message with no signers + let mut qc4 = qc.clone(); + qc4.map.insert( + ReplicaTimeout { + view: qc4.view, + high_vote: None, + high_qc: None, + }, + Signers::new(setup.genesis.validators.len()), + ); + assert_matches!( + qc4.verify(&setup.genesis), + Err(TimeoutQCVerifyError::NoSignersAssigned(_)) + ); + + // QC with overlapping signers + let mut qc5 = qc.clone(); + let mut signers = Signers::new(setup.genesis.validators.len()); + signers + .0 + .set(rng.gen_range(0..setup.genesis.validators.len()), true); + qc5.map.insert( + ReplicaTimeout { + view: qc5.view, + high_vote: None, + high_qc: None, + }, + signers, + ); + assert_matches!( + qc5.verify(&setup.genesis), + Err(TimeoutQCVerifyError::OverlappingSignatureSet(_)) + ); + + // QC with invalid message + let mut qc6 = qc.clone(); + let (mut timeout, signers) = qc6.map.pop_first().unwrap(); + timeout.high_qc = Some(rng.gen()); + qc6.map.insert(timeout, signers); + assert_matches!( + qc6.verify(&setup.genesis), + Err(TimeoutQCVerifyError::InvalidMessage(_, _)) + ); + + // QC with not enough weight + let mut qc7 = qc.clone(); + let (timeout, mut signers) = qc7.map.pop_first().unwrap(); + signers.0.set(0, false); + signers.0.set(4, false); + qc7.map.insert(timeout, signers); + assert_matches!( + qc7.verify(&setup.genesis), + Err(TimeoutQCVerifyError::NotEnoughWeight { .. }) + ); + + // QC with bad signature + let mut qc8 = qc.clone(); + qc8.signature = rng.gen(); + assert_matches!( + qc8.verify(&setup.genesis), + Err(TimeoutQCVerifyError::BadSignature(_)) + ); +} diff --git a/node/libs/roles/src/validator/messages/tests/versions.rs b/node/libs/roles/src/validator/messages/tests/versions.rs new file mode 100644 index 00000000..a19f152a --- /dev/null +++ b/node/libs/roles/src/validator/messages/tests/versions.rs @@ -0,0 +1,102 @@ +use super::*; +use anyhow::Context as _; +use zksync_consensus_crypto::Text; + +mod version1 { + use zksync_consensus_utils::enum_util::Variant as _; + + use super::*; + + /// Note that genesis is NOT versioned by ProtocolVersion. + /// Even if it was, ALL versions of genesis need to be supported FOREVER, + /// unless we introduce dynamic regenesis. + #[test] + fn genesis_hash_change_detector_empty_attesters() { + let want: GenesisHash = Text::new( + "genesis_hash:keccak256:75cfa582fcda9b5da37af8fb63a279f777bb17a97a50519e1a61aad6c77a522f", + ) + .decode() + .unwrap(); + assert_eq!(want, genesis_empty_attesters().hash()); + } + + /// Note that genesis is NOT versioned by ProtocolVersion. + /// Even if it was, ALL versions of genesis need to be supported FOREVER, + /// unless we introduce dynamic regenesis. + #[test] + fn genesis_hash_change_detector_nonempty_attesters() { + let want: GenesisHash = Text::new( + "genesis_hash:keccak256:586a4bc6167c084d7499cead9267b224ab04a4fdeff555630418bcd2df5d186d", + ) + .decode() + .unwrap(); + assert_eq!(want, genesis_with_attesters().hash()); + } + + /// Asserts that msg.hash()==hash and that sig is a + /// valid signature of msg (signed by `keys()[0]`). + #[track_caller] + fn msg_change_detector(msg: Msg, hash: &str, sig: &str) { + let key = validator_keys()[0].clone(); + + (|| { + // Decode hash and signature. + let hash: MsgHash = Text::new(hash).decode()?; + let sig: validator::Signature = Text::new(sig).decode()?; + + // Check if msg.hash() is equal to hash. + if msg.hash() != hash { + anyhow::bail!("Hash mismatch"); + } + + // Check if sig is a valid signature of hash. + sig.verify_hash(&hash, &key.public())?; + + anyhow::Ok(()) + })() + .with_context(|| { + format!( + "\nIntended hash: {:?}\nIntended signature: {:?}", + msg.hash(), + key.sign_hash(&msg.hash()), + ) + }) + .unwrap(); + } + + #[test] + fn replica_commit_change_detector() { + msg_change_detector( + replica_commit().insert(), + "validator_msg:keccak256:ccbb11a6b3f4e06840a2a06abc2a245a2b3de30bb951e759a9ec6920f74f0632", + "validator:signature:bls12_381:8e41b89c89c0de8f83102966596ab95f6bdfdc18fceaceb224753b3ff495e02d5479c709829bd6d0802c5a1f24fa96b5", + ); + } + + #[test] + fn replica_new_view_change_detector() { + msg_change_detector( + replica_new_view().insert(), + "validator_msg:keccak256:2be143114cd3442b96d5f6083713c4c338a1c18ef562ede4721ebf037689a6ad", + "validator:signature:bls12_381:9809b66d44509cf7847baaa03a35ae87062f9827cf1f90c8353f057eee45b79fde0f4c4c500980b69c59263b51b6d072", + ); + } + + #[test] + fn replica_timeout_change_detector() { + msg_change_detector( + replica_timeout().insert(), + "validator_msg:keccak256:615fa6d2960b48e30ab88fe195bbad161b8a6f9a59a45ca86b5e2f20593f76cd", + "validator:signature:bls12_381:ac9b6d340bf1b04421455676b8a28a8de079cd9b40f75f1009aa3da32981690bc520d4ec0284ae030fc8b036d86ca307", + ); + } + + #[test] + fn leader_proposal_change_detector() { + msg_change_detector( + leader_proposal().insert(), + "validator_msg:keccak256:4c1b2cf1e8fbb00cde86caee200491df15c45d5c88402e227c1f3e1b416c4255", + "validator:signature:bls12_381:81f865807067c6f70f17f9716e6d41c0103c2366abb6721408fb7d27ead6332798bd7b34d5f4a63e324082586b2c69a3", + ); + } +} diff --git a/node/libs/roles/src/validator/testonly.rs b/node/libs/roles/src/validator/testonly.rs index 39c4f8ce..5ceff2ac 100644 --- a/node/libs/roles/src/validator/testonly.rs +++ b/node/libs/roles/src/validator/testonly.rs @@ -2,9 +2,10 @@ use super::{ AggregateSignature, Block, BlockHeader, BlockNumber, ChainId, CommitQC, Committee, ConsensusMsg, FinalBlock, ForkNumber, Genesis, GenesisHash, GenesisRaw, Justification, - LeaderCommit, LeaderPrepare, Msg, MsgHash, NetAddress, Payload, PayloadHash, Phase, - PreGenesisBlock, PrepareQC, ProofOfPossession, ProtocolVersion, PublicKey, ReplicaCommit, - ReplicaPrepare, SecretKey, Signature, Signed, Signers, View, ViewNumber, WeightedValidator, + LeaderProposal, Msg, MsgHash, NetAddress, Payload, PayloadHash, Phase, PreGenesisBlock, + ProofOfPossession, ProposalJustification, ProtocolVersion, PublicKey, ReplicaCommit, + ReplicaNewView, ReplicaTimeout, SecretKey, Signature, Signed, Signers, TimeoutQC, View, + ViewNumber, WeightedValidator, }; use crate::{attester, validator::LeaderSelectionMode}; use bit_vec::BitVec; @@ -37,10 +38,6 @@ pub struct SetupSpec { pub leader_selection: LeaderSelectionMode, } -/// Test setup. -#[derive(Debug, Clone)] -pub struct Setup(SetupInner); - impl SetupSpec { /// New `SetupSpec`. pub fn new(rng: &mut impl Rng, validators: usize) -> Self { @@ -67,6 +64,10 @@ impl SetupSpec { } } +/// Test setup. +#[derive(Debug, Clone)] +pub struct Setup(SetupInner); + impl Setup { /// New `Setup`. pub fn new(rng: &mut impl Rng, validators: usize) -> Self { @@ -80,6 +81,51 @@ impl Setup { Self::from_spec(rng, spec) } + /// Generates a new `Setup` from the given `SetupSpec`. + pub fn from_spec(rng: &mut impl Rng, spec: SetupSpec) -> Self { + let mut this = Self(SetupInner { + genesis: GenesisRaw { + chain_id: spec.chain_id, + fork_number: spec.fork_number, + first_block: spec.first_block, + + protocol_version: spec.protocol_version, + validators: Committee::new(spec.validator_weights.iter().map(|(k, w)| { + WeightedValidator { + key: k.public(), + weight: *w, + } + })) + .unwrap(), + attesters: attester::Committee::new(spec.attester_weights.iter().map(|(k, w)| { + attester::WeightedAttester { + key: k.public(), + weight: *w, + } + })) + .unwrap() + .into(), + leader_selection: spec.leader_selection, + } + .with_hash(), + validator_keys: spec.validator_weights.into_iter().map(|(k, _)| k).collect(), + attester_keys: spec.attester_weights.into_iter().map(|(k, _)| k).collect(), + blocks: vec![], + }); + // Populate pregenesis blocks. + for block in spec.first_pregenesis_block.0..spec.first_block.0 { + this.0.blocks.push( + PreGenesisBlock { + number: BlockNumber(block), + payload: rng.gen(), + justification: rng.gen(), + } + .into(), + ); + } + this + } + /// Next block to finalize. pub fn next(&self) -> BlockNumber { match self.0.blocks.last() { @@ -143,52 +189,107 @@ impl Setup { let first = self.0.blocks.first()?.number(); self.0.blocks.get(n.0.checked_sub(first.0)? as usize) } -} -impl Setup { - /// Generates a new `Setup` from the given `SetupSpec`. - pub fn from_spec(rng: &mut impl Rng, spec: SetupSpec) -> Self { - let mut this = Self(SetupInner { - genesis: GenesisRaw { - chain_id: spec.chain_id, - fork_number: spec.fork_number, - first_block: spec.first_block, + /// Creates a View with the given view number. + pub fn make_view(&self, number: ViewNumber) -> View { + View { + genesis: self.genesis.hash(), + number, + } + } - protocol_version: spec.protocol_version, - validators: Committee::new(spec.validator_weights.iter().map(|(k, w)| { - WeightedValidator { - key: k.public(), - weight: *w, - } - })) - .unwrap(), - attesters: attester::Committee::new(spec.attester_weights.iter().map(|(k, w)| { - attester::WeightedAttester { - key: k.public(), - weight: *w, - } - })) - .unwrap() - .into(), - leader_selection: spec.leader_selection, - } - .with_hash(), - validator_keys: spec.validator_weights.into_iter().map(|(k, _)| k).collect(), - attester_keys: spec.attester_weights.into_iter().map(|(k, _)| k).collect(), - blocks: vec![], - }); - // Populate pregenesis blocks. - for block in spec.first_pregenesis_block.0..spec.first_block.0 { - this.0.blocks.push( - PreGenesisBlock { - number: BlockNumber(block), - payload: rng.gen(), - justification: rng.gen(), - } - .into(), - ); + /// Creates a ReplicaCommit with a random payload. + pub fn make_replica_commit(&self, rng: &mut impl Rng, view: ViewNumber) -> ReplicaCommit { + ReplicaCommit { + view: self.make_view(view), + proposal: BlockHeader { + number: self.next(), + payload: rng.gen(), + }, } - this + } + + /// Creates a ReplicaCommit with the given payload. + pub fn make_replica_commit_with_payload( + &self, + payload: &Payload, + view: ViewNumber, + ) -> ReplicaCommit { + ReplicaCommit { + view: self.make_view(view), + proposal: BlockHeader { + number: self.next(), + payload: payload.hash(), + }, + } + } + + /// Creates a CommitQC with a random payload. + pub fn make_commit_qc(&self, rng: &mut impl Rng, view: ViewNumber) -> CommitQC { + let mut qc = CommitQC::new(self.make_replica_commit(rng, view), &self.genesis); + for key in &self.validator_keys { + qc.add(&key.sign_msg(qc.message.clone()), &self.genesis) + .unwrap(); + } + qc + } + + /// Creates a CommitQC with the given payload. + pub fn make_commit_qc_with_payload(&self, payload: &Payload, view: ViewNumber) -> CommitQC { + let mut qc = CommitQC::new( + self.make_replica_commit_with_payload(payload, view), + &self.genesis, + ); + for key in &self.validator_keys { + qc.add(&key.sign_msg(qc.message.clone()), &self.genesis) + .unwrap(); + } + qc + } + + /// Creates a ReplicaTimeout with a random payload. + pub fn make_replica_timeout(&self, rng: &mut impl Rng, view: ViewNumber) -> ReplicaTimeout { + let high_vote_view = ViewNumber(rng.gen_range(0..=view.0)); + let high_qc_view = ViewNumber(rng.gen_range(0..high_vote_view.0)); + ReplicaTimeout { + view: self.make_view(view), + high_vote: Some(self.make_replica_commit(rng, high_vote_view)), + high_qc: Some(self.make_commit_qc(rng, high_qc_view)), + } + } + + /// Creates a TimeoutQC. If a payload is given, the QC will contain a + /// re-proposal for that payload + pub fn make_timeout_qc( + &self, + rng: &mut impl Rng, + view: ViewNumber, + payload_opt: Option<&Payload>, + ) -> TimeoutQC { + let mut vote = if let Some(payload) = payload_opt { + self.make_replica_commit_with_payload(payload, view.prev().unwrap()) + } else { + self.make_replica_commit(rng, view.prev().unwrap()) + }; + let commit_qc = match self.0.blocks.last().unwrap() { + Block::Final(block) => block.justification.clone(), + _ => unreachable!(), + }; + + let mut qc = TimeoutQC::new(self.make_view(view)); + if payload_opt.is_none() { + vote.proposal.payload = rng.gen(); + } + let msg = ReplicaTimeout { + view: self.make_view(view), + high_vote: Some(vote.clone()), + high_qc: Some(commit_qc.clone()), + }; + for key in &self.validator_keys { + qc.add(&key.sign_msg(msg.clone()), &self.genesis).unwrap(); + } + + qc } } @@ -392,9 +493,9 @@ impl Distribution for Standard { } } -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> ReplicaPrepare { - ReplicaPrepare { +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> ReplicaTimeout { + ReplicaTimeout { view: rng.gen(), high_vote: rng.gen(), high_qc: rng.gen(), @@ -411,30 +512,29 @@ impl Distribution for Standard { } } -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> LeaderPrepare { - LeaderPrepare { - proposal: rng.gen(), - proposal_payload: rng.gen(), +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> ReplicaNewView { + ReplicaNewView { justification: rng.gen(), } } } -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> LeaderCommit { - LeaderCommit { +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> LeaderProposal { + LeaderProposal { + proposal_payload: rng.gen(), justification: rng.gen(), } } } -impl Distribution for Standard { - fn sample(&self, rng: &mut R) -> PrepareQC { +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> TimeoutQC { let n = rng.gen_range(1..11); let map = (0..n).map(|_| (rng.gen(), rng.gen())).collect(); - PrepareQC { + TimeoutQC { view: rng.gen(), map, signature: rng.gen(), @@ -452,6 +552,16 @@ impl Distribution for Standard { } } +impl Distribution for Standard { + fn sample(&self, rng: &mut R) -> ProposalJustification { + match rng.gen_range(0..2) { + 0 => ProposalJustification::Commit(rng.gen()), + 1 => ProposalJustification::Timeout(rng.gen()), + _ => unreachable!(), + } + } +} + impl Distribution for Standard { fn sample(&self, rng: &mut R) -> Signers { Signers(BitVec::from_bytes(&rng.gen::<[u8; 4]>())) @@ -523,10 +633,10 @@ impl Distribution for Standard { impl Distribution for Standard { fn sample(&self, rng: &mut R) -> ConsensusMsg { match rng.gen_range(0..4) { - 0 => ConsensusMsg::ReplicaPrepare(rng.gen()), + 0 => ConsensusMsg::LeaderProposal(rng.gen()), 1 => ConsensusMsg::ReplicaCommit(rng.gen()), - 2 => ConsensusMsg::LeaderPrepare(rng.gen()), - 3 => ConsensusMsg::LeaderCommit(rng.gen()), + 2 => ConsensusMsg::ReplicaNewView(rng.gen()), + 3 => ConsensusMsg::ReplicaTimeout(rng.gen()), _ => unreachable!(), } } diff --git a/node/libs/roles/src/validator/tests.rs b/node/libs/roles/src/validator/tests.rs index eafa3a9e..95d0f190 100644 --- a/node/libs/roles/src/validator/tests.rs +++ b/node/libs/roles/src/validator/tests.rs @@ -1,11 +1,8 @@ use super::*; -use crate::validator::testonly::Setup; -use assert_matches::assert_matches; -use rand::{seq::SliceRandom, Rng}; -use std::vec; +use rand::Rng; use zksync_concurrency::ctx; use zksync_consensus_crypto::{ByteFmt, Text, TextFmt}; -use zksync_protobuf::{testonly::test_encode_random, ProtoFmt}; +use zksync_protobuf::testonly::test_encode_random; #[test] fn test_byte_encoding() { @@ -90,7 +87,7 @@ fn test_schema_encoding() { test_encode_random::(rng); test_encode_random::(rng); test_encode_random::>(rng); - test_encode_random::(rng); + test_encode_random::(rng); test_encode_random::(rng); test_encode_random::(rng); test_encode_random::(rng); @@ -102,432 +99,3 @@ fn test_schema_encoding() { test_encode_random::(rng); test_encode_random::(rng); } - -#[test] -fn test_genesis_verify() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let genesis = Setup::new(rng, 1).genesis.clone(); - assert!(genesis.verify().is_ok()); - assert!(Genesis::read(&genesis.build()).is_ok()); - - let mut genesis = (*genesis).clone(); - genesis.leader_selection = LeaderSelectionMode::Sticky(rng.gen()); - let genesis = genesis.with_hash(); - assert!(genesis.verify().is_err()); - assert!(Genesis::read(&genesis.build()).is_err()) -} - -#[test] -fn test_signature_verify() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let msg1: MsgHash = rng.gen(); - let msg2: MsgHash = rng.gen(); - - let key1: SecretKey = rng.gen(); - let key2: SecretKey = rng.gen(); - - let sig1 = key1.sign_hash(&msg1); - - // Matching key and message. - sig1.verify_hash(&msg1, &key1.public()).unwrap(); - - // Mismatching message. - assert!(sig1.verify_hash(&msg2, &key1.public()).is_err()); - - // Mismatching key. - assert!(sig1.verify_hash(&msg1, &key2.public()).is_err()); -} - -#[test] -fn test_agg_signature_verify() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let msg1: MsgHash = rng.gen(); - let msg2: MsgHash = rng.gen(); - - let key1: SecretKey = rng.gen(); - let key2: SecretKey = rng.gen(); - - let sig1 = key1.sign_hash(&msg1); - let sig2 = key2.sign_hash(&msg2); - - let agg_sig = AggregateSignature::aggregate(vec![&sig1, &sig2]); - - // Matching key and message. - agg_sig - .verify_hash([(msg1, &key1.public()), (msg2, &key2.public())].into_iter()) - .unwrap(); - - // Mismatching message. - assert!(agg_sig - .verify_hash([(msg2, &key1.public()), (msg1, &key2.public())].into_iter()) - .is_err()); - - // Mismatching key. - assert!(agg_sig - .verify_hash([(msg1, &key2.public()), (msg2, &key1.public())].into_iter()) - .is_err()); -} - -fn make_view(number: ViewNumber, setup: &Setup) -> View { - View { - genesis: setup.genesis.hash(), - number, - } -} - -fn make_replica_commit(rng: &mut impl Rng, view: ViewNumber, setup: &Setup) -> ReplicaCommit { - ReplicaCommit { - view: make_view(view, setup), - proposal: rng.gen(), - } -} - -fn make_commit_qc(rng: &mut impl Rng, view: ViewNumber, setup: &Setup) -> CommitQC { - let mut qc = CommitQC::new(make_replica_commit(rng, view, setup), &setup.genesis); - for key in &setup.validator_keys { - qc.add(&key.sign_msg(qc.message.clone()), &setup.genesis) - .unwrap(); - } - qc -} - -fn make_replica_prepare(rng: &mut impl Rng, view: ViewNumber, setup: &Setup) -> ReplicaPrepare { - ReplicaPrepare { - view: make_view(view, setup), - high_vote: { - let view = ViewNumber(rng.gen_range(0..view.0)); - Some(make_replica_commit(rng, view, setup)) - }, - high_qc: { - let view = ViewNumber(rng.gen_range(0..view.0)); - Some(make_commit_qc(rng, view, setup)) - }, - } -} - -#[test] -fn test_commit_qc() { - use CommitQCVerifyError as Error; - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - // This will create equally weighted validators - let setup1 = Setup::new(rng, 6); - let setup2 = Setup::new(rng, 6); - let mut genesis3 = (*setup1.genesis).clone(); - genesis3.validators = - Committee::new(setup1.genesis.validators.iter().take(3).cloned()).unwrap(); - let genesis3 = genesis3.with_hash(); - - for i in 0..setup1.validator_keys.len() + 1 { - let view = rng.gen(); - let mut qc = CommitQC::new(make_replica_commit(rng, view, &setup1), &setup1.genesis); - for key in &setup1.validator_keys[0..i] { - qc.add(&key.sign_msg(qc.message.clone()), &setup1.genesis) - .unwrap(); - } - let expected_weight: u64 = setup1 - .genesis - .validators - .iter() - .take(i) - .map(|w| w.weight) - .sum(); - if expected_weight >= setup1.genesis.validators.threshold() { - qc.verify(&setup1.genesis).unwrap(); - } else { - assert_matches!( - qc.verify(&setup1.genesis), - Err(Error::NotEnoughSigners { .. }) - ); - } - - // Mismatching validator sets. - assert!(qc.verify(&setup2.genesis).is_err()); - assert!(qc.verify(&genesis3).is_err()); - } -} - -#[test] -fn test_commit_qc_add_errors() { - use CommitQCAddError as Error; - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let setup = Setup::new(rng, 2); - let view = rng.gen(); - let mut qc = CommitQC::new(make_replica_commit(rng, view, &setup), &setup.genesis); - let msg = qc.message.clone(); - // Add the message - assert_matches!( - qc.add( - &setup.validator_keys[0].sign_msg(msg.clone()), - &setup.genesis - ), - Ok(()) - ); - - // Try to add a message for a different view - let mut msg1 = msg.clone(); - msg1.view.number = view.next(); - assert_matches!( - qc.add(&setup.validator_keys[0].sign_msg(msg1), &setup.genesis), - Err(Error::InconsistentMessages { .. }) - ); - - // Try to add a message from a signer not in committee - assert_matches!( - qc.add( - &rng.gen::().sign_msg(msg.clone()), - &setup.genesis - ), - Err(Error::SignerNotInCommittee { .. }) - ); - - // Try to add the same message already added by same validator - assert_matches!( - qc.add( - &setup.validator_keys[0].sign_msg(msg.clone()), - &setup.genesis - ), - Err(Error::Exists { .. }) - ); - - // Add same message signed by another validator. - assert_matches!( - qc.add(&setup.validator_keys[1].sign_msg(msg), &setup.genesis), - Ok(()) - ); -} - -#[test] -fn test_prepare_qc() { - use PrepareQCVerifyError as Error; - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - // This will create equally weighted validators - let setup1 = Setup::new(rng, 6); - let setup2 = Setup::new(rng, 6); - let mut genesis3 = (*setup1.genesis).clone(); - genesis3.validators = - Committee::new(setup1.genesis.validators.iter().take(3).cloned()).unwrap(); - let genesis3 = genesis3.with_hash(); - - let view: ViewNumber = rng.gen(); - let msgs: Vec<_> = (0..3) - .map(|_| make_replica_prepare(rng, view, &setup1)) - .collect(); - - for n in 0..setup1.validator_keys.len() + 1 { - let mut qc = PrepareQC::new(msgs[0].view.clone()); - for key in &setup1.validator_keys[0..n] { - qc.add( - &key.sign_msg(msgs.choose(rng).unwrap().clone()), - &setup1.genesis, - ) - .unwrap(); - } - let expected_weight: u64 = setup1 - .genesis - .validators - .iter() - .take(n) - .map(|w| w.weight) - .sum(); - if expected_weight >= setup1.genesis.validators.threshold() { - qc.verify(&setup1.genesis).unwrap(); - } else { - assert_matches!( - qc.verify(&setup1.genesis), - Err(Error::NotEnoughSigners { .. }) - ); - } - - // Mismatching validator sets. - assert!(qc.verify(&setup2.genesis).is_err()); - assert!(qc.verify(&genesis3).is_err()); - } -} - -#[test] -fn test_prepare_qc_high_vote() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - // This will create equally weighted validators - let setup = Setup::new(rng, 6); - - let view_num: ViewNumber = rng.gen(); - let msg_a = make_replica_prepare(rng, view_num, &setup); - let msg_b = make_replica_prepare(rng, view_num, &setup); - let msg_c = make_replica_prepare(rng, view_num, &setup); - - // Case with 1 subquorum. - let mut qc = PrepareQC::new(msg_a.view.clone()); - - for key in &setup.validator_keys { - qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) - .unwrap(); - } - - assert!(qc.high_vote(&setup.genesis).is_some()); - - // Case with 2 subquorums. - let mut qc = PrepareQC::new(msg_a.view.clone()); - - for key in &setup.validator_keys[0..3] { - qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) - .unwrap(); - } - - for key in &setup.validator_keys[3..6] { - qc.add(&key.sign_msg(msg_b.clone()), &setup.genesis) - .unwrap(); - } - - assert!(qc.high_vote(&setup.genesis).is_none()); - - // Case with no subquorums. - let mut qc = PrepareQC::new(msg_a.view.clone()); - - for key in &setup.validator_keys[0..2] { - qc.add(&key.sign_msg(msg_a.clone()), &setup.genesis) - .unwrap(); - } - - for key in &setup.validator_keys[2..4] { - qc.add(&key.sign_msg(msg_b.clone()), &setup.genesis) - .unwrap(); - } - - for key in &setup.validator_keys[4..6] { - qc.add(&key.sign_msg(msg_c.clone()), &setup.genesis) - .unwrap(); - } - - assert!(qc.high_vote(&setup.genesis).is_none()); -} - -#[test] -fn test_prepare_qc_add_errors() { - use PrepareQCAddError as Error; - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - let setup = Setup::new(rng, 2); - let view = rng.gen(); - let msg = make_replica_prepare(rng, view, &setup); - let mut qc = PrepareQC::new(msg.view.clone()); - let msg = make_replica_prepare(rng, view, &setup); - - // Add the message - assert_matches!( - qc.add( - &setup.validator_keys[0].sign_msg(msg.clone()), - &setup.genesis - ), - Ok(()) - ); - - // Try to add a message for a different view - let mut msg1 = msg.clone(); - msg1.view.number = view.next(); - assert_matches!( - qc.add(&setup.validator_keys[0].sign_msg(msg1), &setup.genesis), - Err(Error::InconsistentViews { .. }) - ); - - // Try to add a message from a signer not in committee - assert_matches!( - qc.add( - &rng.gen::().sign_msg(msg.clone()), - &setup.genesis - ), - Err(Error::SignerNotInCommittee { .. }) - ); - - // Try to add the same message already added by same validator - assert_matches!( - qc.add( - &setup.validator_keys[0].sign_msg(msg.clone()), - &setup.genesis - ), - Err(Error::Exists { .. }) - ); - - // Try to add a message for a validator that already added another message - let msg2 = make_replica_prepare(rng, view, &setup); - assert_matches!( - qc.add(&setup.validator_keys[0].sign_msg(msg2), &setup.genesis), - Err(Error::Exists { .. }) - ); - - // Add same message signed by another validator. - assert_matches!( - qc.add( - &setup.validator_keys[1].sign_msg(msg.clone()), - &setup.genesis - ), - Ok(()) - ); -} - -#[test] -fn test_validator_committee_weights() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - // Validators with non-uniform weights - let setup = Setup::new_with_weights(rng, vec![1000, 600, 800, 6000, 900, 700]); - // Expected sum of the validators weights - let sums = [1000, 1600, 2400, 8400, 9300, 10000]; - - let view: ViewNumber = rng.gen(); - let msg = make_replica_prepare(rng, view, &setup); - let mut qc = PrepareQC::new(msg.view.clone()); - for (n, weight) in sums.iter().enumerate() { - let key = &setup.validator_keys[n]; - qc.add(&key.sign_msg(msg.clone()), &setup.genesis).unwrap(); - let signers = &qc.map[&msg]; - assert_eq!(setup.genesis.validators.weight(signers), *weight); - } -} - -#[test] -fn test_committee_weights_overflow_check() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let validators: Vec = [u64::MAX / 5; 6] - .iter() - .map(|w| WeightedValidator { - key: rng.gen::().public(), - weight: *w, - }) - .collect(); - - // Creation should overflow - assert_matches!(Committee::new(validators), Err(_)); -} - -#[test] -fn test_committee_with_zero_weights() { - let ctx = ctx::test_root(&ctx::RealClock); - let rng = &mut ctx.rng(); - - let validators: Vec = [1000, 0, 800, 6000, 0, 700] - .iter() - .map(|w| WeightedValidator { - key: rng.gen::().public(), - weight: *w, - }) - .collect(); - - // Committee creation should error on zero weight validators - assert_matches!(Committee::new(validators), Err(_)); -} diff --git a/node/libs/storage/src/proto/mod.proto b/node/libs/storage/src/proto/mod.proto index e06e84da..50744fd3 100644 --- a/node/libs/storage/src/proto/mod.proto +++ b/node/libs/storage/src/proto/mod.proto @@ -13,6 +13,8 @@ message ReplicaState { optional uint64 view = 1; // required; ViewNumber optional roles.validator.Phase phase = 2; // required optional roles.validator.ReplicaCommit high_vote = 3; // optional + // TODO: name should be high_commit_qc optional roles.validator.CommitQC high_qc = 4; // optional repeated Proposal proposals = 5; + optional roles.validator.TimeoutQC high_timeout_qc = 6; // optional } diff --git a/node/libs/storage/src/replica_store.rs b/node/libs/storage/src/replica_store.rs index 465d26d6..da7698ff 100644 --- a/node/libs/storage/src/replica_store.rs +++ b/node/libs/storage/src/replica_store.rs @@ -41,7 +41,9 @@ pub struct ReplicaState { /// The highest block proposal that the replica has committed to. pub high_vote: Option, /// The highest commit quorum certificate known to the replica. - pub high_qc: Option, + pub high_commit_qc: Option, + /// The highest timeout quorum certificate known to the replica. + pub high_timeout_qc: Option, /// A cache of the received block proposals. pub proposals: Vec, } @@ -52,7 +54,8 @@ impl Default for ReplicaState { view: validator::ViewNumber(0), phase: validator::Phase::Prepare, high_vote: None, - high_qc: None, + high_commit_qc: None, + high_timeout_qc: None, proposals: vec![], } } @@ -84,7 +87,8 @@ impl ProtoFmt for ReplicaState { view: validator::ViewNumber(r.view.context("view_number")?), phase: read_required(&r.phase).context("phase")?, high_vote: read_optional(&r.high_vote).context("high_vote")?, - high_qc: read_optional(&r.high_qc).context("high_qc")?, + high_commit_qc: read_optional(&r.high_qc).context("high_commit_qc")?, + high_timeout_qc: read_optional(&r.high_timeout_qc).context("high_timeout_qc")?, proposals: r .proposals .iter() @@ -99,7 +103,8 @@ impl ProtoFmt for ReplicaState { view: Some(self.view.0), phase: Some(self.phase.build()), high_vote: self.high_vote.as_ref().map(|x| x.build()), - high_qc: self.high_qc.as_ref().map(|x| x.build()), + high_qc: self.high_commit_qc.as_ref().map(|x| x.build()), + high_timeout_qc: self.high_timeout_qc.as_ref().map(|x| x.build()), proposals: self.proposals.iter().map(|p| p.build()).collect(), } } diff --git a/node/libs/storage/src/testonly/mod.rs b/node/libs/storage/src/testonly/mod.rs index c36d3b74..e52293ea 100644 --- a/node/libs/storage/src/testonly/mod.rs +++ b/node/libs/storage/src/testonly/mod.rs @@ -26,7 +26,8 @@ impl Distribution for Standard { view: rng.gen(), phase: rng.gen(), high_vote: rng.gen(), - high_qc: rng.gen(), + high_commit_qc: rng.gen(), + high_timeout_qc: rng.gen(), proposals: (0..rng.gen_range(1..11)).map(|_| rng.gen()).collect(), } } diff --git a/spec/README.md b/spec/README.md index 2ef8e82f..4380820a 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,3 +1,34 @@ -# ChonkyBFT's Specification +# ChonkyBFT -This is a formal specification of ChonkyBFT consensus protocol in Quint. +This folder contains the specification of the ChonkyBFT, a new consensus protocol created by Bruno França and Grzegorz Prusak at Matter Labs. It has both the pseudo-code specification that was used as the basis for the Rust implementation in the rest of this repo and the Quint specification that was used to formally verify the protocol. +Chonky BFT is a consensus protocol inspired by [FaB Paxos](https://www.cs.cornell.edu/lorenzo/papers/Martin06Fast.pdf), [Fast-HotStuff](https://arxiv.org/abs/2010.11454) and [HotStuff-2](https://eprint.iacr.org/2023/397). +It is committee-based and has only one round of voting, single slot finality, quadratic communication and _n=5f+1_ fault tolerance. Let's discuss what were our objectives when designing ChonkyBFT. + +## Design goals in practice vs. theory + +We find that most recent research on consensus algorithms unfortunately has become somewhat detached from the realities of running those same consensus algorithms in practice. This has led to researchers optimizing algorithms along the wrong dimensions. Many times we see tables in papers comparing different algorithms along metrics that genuinely don’t matter when those algorithms are implemented. + +### What doesn’t matter + +- Authenticator complexity: This is probably the worst one. Optimizing to have fewer signatures made sense decades ago when crypto operations were expensive. Today, digital signatures are fast and small. However, many papers (for example HotStuff) still report this measure and even go as far as suggesting threshold signatures over multisignatures, which introduces a much more complex step of distributed key generation instead of spending some more milliseconds on verifying the signatures. +- Message complexity: This also tends to be a red herring. In theory, the fewer messages are passed around the network, the faster the algorithm will be. In practice, it depends on where the bottleneck is. If your algorithm has linear communication, but the leader still has to send and receive N messages, then you are not gaining any meaningful performance. This also has the unfortunate effect of treating every message the same, while in practice a block proposal can be megabytes long and a block commit is a few kilobytes at most. +- Block latency: This is the wrong latency to consider. It doesn’t matter if our block time is 0.1s, if then we have to wait 100 blocks to finalize. All it matters is how long it takes for an user to see their transaction finalized. This has led to algorithms like Narwhal and Tusk, which claim to have just one round of voting but another round “hidden” in the block broadcast mechanism. This actually leads to a worse latency for the user, even though the block times are shorter. + +### What does matter + +- Systemic complexity: This relates to the [systemic vs. encapsulated complexity](https://vitalik.eth.limo/general/2022/02/28/complexity.html) topic. Our consensus algorithms are not run in isolation, they are meant to support other applications. An example of this problem is probabilistic vs provable finality. Algorithms that finalize probabilistically impose complexity on the applications. Exchanges must determine how many confirmations to wait for each different chain they accept, the same for multi-chain dapps, hybrid dapps, block explorers, wallets, etc. Algorithms that finalize provably give a clear signal to every application that they can use. This is important enough that even Ethereum is planning to move to [single-slot finality](https://ethereum.org/en/roadmap/single-slot-finality/#why-aim-for-quicker-finality), because not finalizing every block is not enough. +- Simplicity: To model and implement the algorithm. Your algorithm might be able to save one round-trip in an optimistic scenario, but is it worth it if it’s too complex to create a formal model out of it? And if then the implementation will take 4 engineers and 3 audits? Simple algorithms that can be formally proven and are straight-forward to implement are more secure algorithms. A bug that causes downtime (or even worse, safety violations) is much worse for the UX than slightly slower block times. +- Transaction latency: What was discussed before. The only latency that matters is the one experienced by the user. + +## Lessons learned + +For our particular use case, there are a few lessons that we learned from researching and implementing previous consensus algorithms: + +- Chained consensus is not worth it. It doesn’t improve the throughput or the latency while increasing systemic complexity. We always finalize every block. +- Lower fault tolerance to reduce voting rounds. This we learned from FaB Paxos. Decreasing our fault tolerance from *3f+1* to *5f+1* allows us to finalize in just one voting round. +- Linear communication is not worth it. Quadratic communication for replicas simplifies security (there are fewer cases where we need to consider the effect of a malicious leader), implementation (you can fully separate the leader component) and view changes (constant timeouts are enough, [Jolteon/Ditto](https://arxiv.org/abs/2106.10362) ended up going in that direction after trying to implement HotStuff). Further, the performance drop is likely not significant (see [ParBFT](https://eprint.iacr.org/2023/679.pdf)). +- Re-proposals as a way of guaranteeing that there are no “rogue” blocks. This is a problem that didn’t get any attention so far (as far as we know), and is probably somewhat unique to public blockchains. The issue is that in all committee-based consensus algorithms it is possible that a commit QC (to use HotStuff’s terminology) is formed but that not enough replicas receive it. This will cause a timeout and another block to be proposed. Most algorithms just solve this by saying that the old block is no longer valid. All honest replicas will be in agreement about which block is canonical, but someone who just receives that single block and is not aware of the timeout will think that that particular block was finalized. This breaks the very desirable property of being able to verify that a given block is part of the chain just from seeing the block, without being required to have the entire chain. The way we solve this is to require that block proposals after a timeout (where a commit QC might have been formed) re-propose the previous block. This guarantees that if we see a block with a valid commit QC, then that block is part of the chain (maybe it wasn’t finalized in that particular view, but it was certainly finalized). +- Always justify messages to remove time dependencies. That’s something we got from Fast-HotStuff. Messages should have enough information by themselves that any replica is capable of verifying their validity without any other information (with the exception of having previous blocks, but that’s external to the consensus algorithm anyway). If we don’t, then we introduce subtle timing dependencies. For example, Tendermint had a bug that was only discovered years later, where the solution was that the leader had to wait for the maximum network delay at the end of every round. If that wait doesn’t happen, a lock can occur. Funnily enough, Hotstuff-2 reintroduces this timing dependency in order to get rid of one round-trip, which significantly worsens the difficulty of modelling and implementing such a system. +- Make garbage collection and reconfiguration part of the algorithm. These are parts of the algorithm that will certainly be implemented. If we don’t specify and model them before, we will be left with awkwardly implementing them later on. + +FaB Paxos satisfies the first 4 points and Fast-HotStuff satisfies the 5th. ChonkyBFT is basically FaB Paxos with some ideas from Fast-HotStuff/HotStuff-2. \ No newline at end of file diff --git a/spec/informal-spec/README.md b/spec/informal-spec/README.md index b095cb38..94681c82 100644 --- a/spec/informal-spec/README.md +++ b/spec/informal-spec/README.md @@ -1,11 +1,13 @@ -# ChonkyBFT Specification +# ChonkyBFT Informal Specification -This is a ChonkyBFT specification in pseudocode. +This is the ChonkyBFT specification in pseudocode. + +We’ll assume there’s a static set of nodes. Each node has 3 components: replica, proposer and fetcher. They are modeled as concurrent tasks or actors. Proposer and fetcher can read the replica state, but can’t write to it. There's a couple of considerations that are not described in the pseudo-code: - **Network model**. Messages might be delivered out of order, but we don’t guarantee eventual delivery for *all* messages. Actually, our network only guarantees eventual delivery of the most recent message for each type. That’s because each replica only stores the last outgoing message of each type in memory, and always tries to deliver those messages whenever it reconnects with another replica. - **Garbage collection**. We can’t store all messages, the goal here is to bound the number of messages that each replica stores, in order to avoid DoS attacks. We handle messages like this: - `NewView` messages are never stored, so no garbage collection is necessary. - - We keep all `Proposal` messages until the proposal (or a proposal with the same block number) is finalized (which means any honest replica having both the `Proposal` and the corresponding `CommitQC`, we assume that any honest replica in that situation will immediately broadcast the block on the gossip network). + - We keep all `Proposal` messages until the proposal (or a proposal with the same block number) is finalized (which means any honest replica having both the `Proposal` and the corresponding `CommitQC`, we assume that any honest replica in that situation will immediately broadcast the block on the p2p network. - We only store the newest `CommitVote` **and** `TimeoutVote` for each replica. Honest replicas only change views on QCs, so if they send a newer message, they must also have sent a `NewView` on the transition, which means we can just get the QC from that replica. Even if the other replicas don’t receive the QC, it will just trigger a reproposal. \ No newline at end of file diff --git a/spec/informal-spec/replica.rs b/spec/informal-spec/replica.rs index 5c6e692b..a17b4dc2 100644 --- a/spec/informal-spec/replica.rs +++ b/spec/informal-spec/replica.rs @@ -1,5 +1,6 @@ -// Replica +//! Replica +// This is the state machine that moves the consensus forward. struct ReplicaState { // The view this replica is currently in. view: ViewNumber, @@ -65,15 +66,15 @@ impl ReplicaState { self.high_vote, self.high_commit_qc); - // Update our state so that we can no longer vote commit in this view. - self.phase = Phase::Timeout; + // Update our state so that we can no longer vote commit in this view. + self.phase = Phase::Timeout; - // Send the vote to all replicas (including ourselves). - self.send(vote); + // Send the vote to all replicas (including ourselves). + self.send(vote); } - // Try to get a message from the message queue and process it. We don't - // detail the message queue structure since it's boilerplate. + // Try to get a message from the message queue and process it. We don't + // detail the message queue structure since it's boilerplate. if let Some(message) = message_queue.pop() { match message { Proposal(msg) => { @@ -114,26 +115,25 @@ impl ReplicaState { // As a side result, get the correct block hash. let block_hash = match opt_block_hash { Some(hash) => { - // This is a reproposal. We let the leader repropose blocks without sending - // them in the proposal (it sends only the number + hash). That allows a - // leader to repropose a block without having it stored. - // It is an optimization that allows us to not wait for a leader that has - // the previous proposal stored (which can take 4f views), and to somewhat + // This is a reproposal. + // We let the leader repropose blocks without sending them in the proposal + // (it sends only the block number + block hash). That allows a leader to + // repropose a block without having it stored. Sending reproposals without + // a payload is an optimization that allows us to not wait for a leader that + // has the previous proposal stored (which can take 4f views), and to somewhat // speed up reproposals by skipping block broadcast. // This only saves time because we have a gossip network running in parallel, // and any time a replica is able to create a finalized block (by possessing // both the block and the commit QC) it broadcasts the finalized block (this // was meant to propagate the block to full nodes, but of course validators // will end up receiving it as well). - // However, this can be difficult to model and we might want to just - // ignore the gossip network in the formal model. We will still have liveness - // but in the model we'll end up waiting 4f views to get a leader that has the - // previous block before proposing a new one. This is not that bad, since - // then we can be sure that the consensus will continue even if the gossip - // network is failing for some reason. - // For sanity reasons, we'll check that there's no block in the proposal. - // But this check is completely unnecessary (in theory at least). + // We check that the leader didn't send a payload with the reproposal. + // This isn't technically needed for the consensus to work (it will remain + // safe and live), but it's a good practice to avoid unnecessary data in + // blockchain. + // This unnecessary payload would also effectively be a source of free + // data availability, which the leaders would be incentivized to abuse. assert!(proposal.block.is_none()); hash @@ -169,7 +169,7 @@ impl ReplicaState { self.send(vote); } - // Processed an (already verified) commit_qc received from the network + // Processes a (already verified) commit_qc received from the network // as part of some message. It bumps the local high_commit_qc and if // we have the proposal corresponding to this qc, we append it to the committed_blocks. fn process_commit_qc(&mut self, qc_opt: Option) { @@ -223,7 +223,7 @@ impl ReplicaState { // If the message isn't current, just ignore it. assert!(new_view.view() >= self.view) - // Check that the new view is valid. + // Check that the new view message is valid. assert!(new_view.verify()); // Update our state. diff --git a/spec/informal-spec/types.rs b/spec/informal-spec/types.rs index f5789cff..65989df8 100644 --- a/spec/informal-spec/types.rs +++ b/spec/informal-spec/types.rs @@ -183,8 +183,10 @@ impl SignedTimeoutVote { } fn verify(self) -> bool { - // If we wish, there are two invariants that are easy to check but aren't required for correctness: - // self.view() >= self.high_vote.view() && self.high_vote.view() >= self.high_commit_qc_view + // If we wish, there are three invariants that are easy to check but don't need to be strictly enforced for correctness: + // 1. self.view() >= self.high_vote.view() + // 2. self.high_vote.view() >= self.high_commit_qc_view + // 3. self.view() > self.high_commit_qc_view self.vote.high_commit_qc_view == self.high_commit_qc.map(|x| x.view()) && self.verify_sig() && self.high_commit_qc.map(|qc| qc.verify()) } diff --git a/spec/protocol-spec/README.md b/spec/protocol-spec/README.md index a051d819..2ee8de05 100644 --- a/spec/protocol-spec/README.md +++ b/spec/protocol-spec/README.md @@ -1,4 +1,4 @@ -# ChonkyBFT +# ChonkyBFT Formal Specification This page summarizes the scope of the Quint specification and the experiments we have done with it. This Quint specification was prepared by Igor Konnov and