From 64162f35daf4b7e727388e9723fc299455d41714 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov Date: Thu, 22 Aug 2024 14:50:58 +0300 Subject: [PATCH] Feature: systematic chunks recovery (#2177) * fix: clang-19 warns Signed-off-by: Dmitriy Khaustov aka xDimon * format: remove semicolon Signed-off-by: Dmitriy Khaustov aka xDimon * format: adjust by clang-format-16 Signed-off-by: Dmitriy Khaustov aka xDimon * refactor: warpsync to router Signed-off-by: Dmitriy Khaustov aka xDimon * refactor: keep discovered address longer Signed-off-by: Dmitriy Khaustov aka xDimon * refactor: use only peer_id in protocols Signed-off-by: Dmitriy Khaustov aka xDimon * refactor: separate RequestResponseProtocol to interface and implementation Signed-off-by: Dmitriy Khaustov aka xDimon * refactor: separate some protocols' classes to interface and implementation Signed-off-by: Dmitriy Khaustov aka xDimon * feature: protocols' mocks Signed-off-by: Dmitriy Khaustov aka xDimon * feature: convenient router mock Signed-off-by: Dmitriy Khaustov aka xDimon * update: erasure coding with systematic chunks reconstruction support Signed-off-by: Dmitriy Khaustov aka xDimon * feature: systematic chunks recovery Signed-off-by: Dmitriy Khaustov aka xDimon * retrieve core index Signed-off-by: iceseer * feature: obtaining backing_group for recovery in disputes Signed-off-by: Dmitriy Khaustov aka xDimon * feature: recovery test Signed-off-by: Dmitriy Khaustov aka xDimon * fix: review issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: CI issues Signed-off-by: Dmitriy Khaustov aka xDimon * Fix recovery test Signed-off-by: Igor Egorov * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: remove deprecated clang-tidy rule Signed-off-by: Dmitriy Khaustov aka xDimon * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon * feature: logs for recovery strategies Signed-off-by: Dmitriy Khaustov aka xDimon * feature: logs for recovery strategies Signed-off-by: Dmitriy Khaustov aka xDimon * hotfixe: clang-tidy Signed-off-by: Dmitriy Khaustov aka xDimon * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon * update clang version in dockerfile * ci - change container for clang tidy step * test new tidy workflow * llvm 19 * refactor: add some nolint suppression Signed-off-by: Dmitriy Khaustov aka xDimon * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon --------- Signed-off-by: Dmitriy Khaustov aka xDimon Co-authored-by: iceseer Co-authored-by: Igor Egorov Co-authored-by: Kirill Azovtsev --- .clang-format | 2 +- .clang-tidy | 1 - .github/workflows/test.yml | 82 +- cmake/Hunter/config.cmake | 17 +- cmake/clang-tidy.cmake | 2 +- core/api/service/author/requests/has_key.hpp | 2 +- .../author/requests/has_session_keys.hpp | 2 +- .../service/author/requests/insert_key.hpp | 2 +- .../service/author/requests/rotate_keys.hpp | 2 +- .../requests/submit_and_watch_extrinsic.hpp | 3 +- .../author/requests/submit_extrinsic.hpp | 3 +- .../service/chain/requests/get_block_hash.cpp | 2 +- .../service/child_state/requests/get_keys.hpp | 2 +- .../child_state/requests/get_keys_paged.hpp | 2 +- core/api/service/rpc/requests/methods.hpp | 2 +- core/api/service/state/requests/call.cpp | 2 +- .../service/state/requests/get_keys_paged.hpp | 2 +- .../service/state/requests/query_storage.hpp | 2 +- .../system/requests/account_next_index.hpp | 2 +- core/application/app_configuration.hpp | 4 +- core/application/chain_spec.hpp | 4 +- core/authority_discovery/query/query_impl.cpp | 2 +- core/blockchain/impl/block_tree_impl.cpp | 24 +- core/common/variant_builder.hpp | 2 +- .../grandpa/impl/voting_round_impl.hpp | 4 +- .../grandpa/types/equivocation_proof.hpp | 2 +- .../timeline/impl/consensus_selector_impl.cpp | 4 +- .../consensus/timeline/impl/timeline_impl.cpp | 2 +- core/crypto/key_store.hpp | 10 +- .../secp256k1/secp256k1_provider_impl.cpp | 2 +- core/dispute_coordinator/impl/batch.hpp | 2 +- .../impl/dispute_coordinator_impl.cpp | 15 +- .../impl/dispute_coordinator_impl.hpp | 2 +- core/dispute_coordinator/impl/errors.cpp | 10 +- .../participation/impl/participation_impl.cpp | 17 +- .../participation/impl/participation_impl.hpp | 6 +- core/host_api/impl/host_api_impl.cpp | 6 +- core/injector/application_injector.cpp | 7 +- core/network/impl/peer_manager_impl.cpp | 25 +- .../beefy_justification_protocol.cpp | 3 +- .../beefy_justification_protocol.hpp | 6 +- .../impl/protocols/beefy_protocol_impl.cpp | 4 +- .../impl/protocols/beefy_protocol_impl.hpp | 2 +- .../protocols/block_announce_protocol.cpp | 8 +- .../protocols/block_announce_protocol.hpp | 2 +- .../protocols/fetch_attested_candidate.hpp | 11 +- .../impl/protocols/grandpa_protocol.cpp | 4 +- .../impl/protocols/grandpa_protocol.hpp | 2 +- core/network/impl/protocols/light.cpp | 2 +- core/network/impl/protocols/light.hpp | 6 +- .../impl/protocols/parachain_protocol.hpp | 6 +- .../propagate_transactions_protocol.cpp | 4 +- .../propagate_transactions_protocol.hpp | 2 +- .../protocol_fetch_available_data.hpp | 26 +- .../impl/protocols/protocol_fetch_chunk.hpp | 27 +- .../protocol_fetch_chunk_obsolete.hpp | 20 +- .../impl/protocols/protocol_req_collation.cpp | 14 +- .../impl/protocols/protocol_req_collation.hpp | 2 +- .../impl/protocols/protocol_req_pov.cpp | 40 +- .../impl/protocols/protocol_req_pov.hpp | 4 +- .../protocols/request_response_protocol.hpp | 61 +- .../impl/protocols/send_dispute_protocol.hpp | 39 +- .../impl/protocols/state_protocol_impl.cpp | 18 +- .../impl/protocols/state_protocol_impl.hpp | 2 +- .../impl/protocols/sync_protocol_impl.cpp | 32 +- .../impl/protocols/sync_protocol_impl.hpp | 2 +- core/network/impl/router_libp2p.cpp | 4 + core/network/impl/router_libp2p.hpp | 1 + core/network/impl/stream_engine.cpp | 2 +- .../notifications/connect_and_handshake.hpp | 14 +- core/network/notifications/handshake.hpp | 2 + core/network/protocol_base.hpp | 2 +- core/network/protocols/req_pov_protocol.hpp | 2 +- core/network/req_pov_observer.hpp | 1 + core/network/router.hpp | 3 + core/network/types/grandpa_message.hpp | 2 +- core/network/warp/protocol.hpp | 25 +- .../approval/approval_distribution.cpp | 17 +- .../approval/approval_distribution.hpp | 7 +- core/parachain/approval/state.hpp | 4 +- core/parachain/approval/store.hpp | 3 +- core/parachain/availability/chunks.hpp | 13 + .../availability/fetch/fetch_impl.cpp | 150 ++-- .../availability/recovery/recovery.hpp | 1 + .../availability/recovery/recovery_impl.cpp | 746 ++++++++++++++---- .../availability/recovery/recovery_impl.hpp | 77 +- core/parachain/pvf/pool.cpp | 8 +- core/parachain/pvf/secure_mode_precheck.hpp | 2 +- .../validator/impl/parachain_processor.cpp | 23 +- .../validator/parachain_processor.hpp | 2 +- core/primitives/code_substitutes.hpp | 2 +- core/primitives/strobe.hpp | 2 +- core/primitives/transaction_validity.cpp | 4 +- .../runtime_api/impl/account_nonce_api.cpp | 2 +- core/runtime/wasm_edge/register_host_api.hpp | 4 +- core/scale/std_variant.hpp | 5 +- core/scale/tie.hpp | 6 +- core/scale/tie_hash.hpp | 6 +- core/telemetry/endpoint.hpp | 6 +- core/telemetry/impl/connection_impl.cpp | 4 +- core/utils/kagome_db_editor.cpp | 12 +- core/utils/map.hpp | 6 +- core/utils/struct_to_tuple.hpp | 6 +- core/utils/weak_macro.hpp | 6 +- housekeeping/clang-tidy-diff.sh | 8 +- housekeeping/docker/kagome-dev/Makefile | 50 ++ .../kagome-dev/kagome_builder_deb.Dockerfile | 39 +- .../application/app_state_manager_test.cpp | 6 +- test/core/blockchain/block_tree_test.cpp | 2 +- test/core/consensus/babe/babe_test.cpp | 2 +- .../core/consensus/timeline/timeline_test.cpp | 2 +- test/core/parachain/CMakeLists.txt | 2 + .../parachain/availability/CMakeLists.txt | 14 + .../parachain/availability/recovery_test.cpp | 423 ++++++++++ test/deps/di_test.cpp | 2 +- .../core/authority_discovery/query_mock.hpp | 28 + test/mock/core/network/protocol_base_mock.hpp | 6 +- test/mock/core/network/protocol_mocks.hpp | 52 ++ test/mock/core/network/router_mock.hpp | 77 +- .../parachain/availability_store_mock.hpp | 76 ++ test/testutil/testparam.hpp | 2 +- 121 files changed, 1993 insertions(+), 613 deletions(-) create mode 100644 test/core/parachain/availability/CMakeLists.txt create mode 100644 test/core/parachain/availability/recovery_test.cpp create mode 100644 test/mock/core/authority_discovery/query_mock.hpp create mode 100644 test/mock/core/network/protocol_mocks.hpp create mode 100644 test/mock/core/parachain/availability_store_mock.hpp diff --git a/.clang-format b/.clang-format index 5c94d744b2..2cabb4dd31 100644 --- a/.clang-format +++ b/.clang-format @@ -22,7 +22,7 @@ QualifierAlignment: Custom QualifierOrder: ['inline', 'static', 'constexpr', 'const', 'volatile', 'type', 'restrict'] ReferenceAlignment: Right ReflowComments: true -RemoveSemicolon: true +#RemoveSemicolon: true InsertNewlineAtEOF: true diff --git a/.clang-tidy b/.clang-tidy index b740727818..6aa3e5f949 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -8,7 +8,6 @@ Checks: '-*,clang-analyzer-*,clang-diagnostic-*,readability-*,modernize-*,boost-*,bugprone-*,cppcoreguidelines-*,google-*,hicpp-*,performance-*,readability-*,-google-readability-namespace-comments,-readability-inconsistent-declaration-parameter-name,-readability-braces-around-statements,-hicpp-signed-bitwise,-google-runtime-references,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-readability-magic-numbers,-hicpp-explicit-conversions,-hicpp-uppercase-literal-suffix,-readability-uppercase-literal-suffix,-hicpp-no-array-decay,-hicpp-special-member-functions,-bugprone-narrowing-conversions,-modernize-use-nodiscard,-google-readability-braces-around-statements,-hicpp-braces-around-statements,-bugprone-suspicious-semicolon,-readability-named-parameter,-hicpp-named-parameter,-readability-identifier-naming,-modernize-use-trailing-return-type' WarningsAsErrors: 'modernize-*,cppcoreguidelines-*,boost-*,performance-*,google-build-using-namespace,readability-else-after-return,google-readability-todo' HeaderFilterRegex: '\.(hpp|h)' -AnalyzeTemporaryDtors: false FormatStyle: .clang-format User: bogdan CheckOptions: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 376d74f754..18c655b1b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,6 @@ env: DEFAULT_BUILD_TYPE: Release GIT_REF_NAME: ${{ github.ref_name }} IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }} - CI: true jobs: @@ -132,29 +131,6 @@ jobs: - name: "${{ matrix.options.name }}" run: "${{ matrix.options.run }}" - clang-tidy: - name: "Linux: clang-tidy" - runs-on: ubuntu-latest - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/cache@v4 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: clang-tidy - env: - # build only generated files, so clang-tidy will work correctly - BUILD_FINAL_TARGET: generated - # run build for both WAVM and WasmEdge to download both, otherwise clang-tidy - # isn't able to find headers - run: | - ./housekeeping/make_build.sh - ./housekeeping/clang-tidy-diff.sh - coverage-self-hosted: if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} name: "Self-hosted: Linux: gcc-12 coverage/sonar" @@ -218,6 +194,64 @@ jobs: BUILD_TYPE: "${{ matrix.options.build-type }}" run: ./housekeeping/docker/kagome-dev/make.sh + kagome_dev_docker_build_tidy: + runs-on: ubuntu-latest + timeout-minutes: 180 + name: "Linux: clang-tidy" + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Check version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV + + - name: "Cache dependencies" + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-${{ env.short_commit_hash }} + restore-keys: | + ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy- + + - name: "Build target" + working-directory: ./housekeeping/docker/kagome-dev + run: + make kagome_dev_docker_build_tidy \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ + GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ + CI="true" + + - name: "Cleaning cache" + run: | + find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; + + - name: "Always Save Cache" + id: cache-save + if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') + uses: actions/cache/save@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + kagome_dev_docker_build: runs-on: [ actions-runner-controller ] timeout-minutes: 180 diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index cfcc1ec1a1..36fd7f9b48 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -77,8 +77,8 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM") LLVM VERSION 12.0.1-p4 CMAKE_ARGS - LLVM_ENABLE_PROJECTS=ir - KEEP_PACKAGE_SOURCES + LLVM_ENABLE_PROJECTS=ir + KEEP_PACKAGE_SOURCES ) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -91,7 +91,7 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM") wavm VERSION 1.0.14 CMAKE_ARGS - WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS} + WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS} KEEP_PACKAGE_SOURCES ) endif () @@ -106,5 +106,14 @@ hunter_config( libsecp256k1 VERSION 0.5.1 CMAKE_ARGS - SECP256K1_ENABLE_MODULE_RECOVERY=ON + SECP256K1_ENABLE_MODULE_RECOVERY=ON +) + +hunter_config( + erasure_coding_crust +# VERSION 0.0.8 + URL https://github.com/qdrvm/erasure-coding-crust/archive/refs/tags/v0.0.8.tar.gz + SHA1 6bcdb6327f5da2dcec5c70f2fa63b95a44925af0 + KEEP_PACKAGE_SOURCES ) + diff --git a/cmake/clang-tidy.cmake b/cmake/clang-tidy.cmake index 0ef5cbb963..c738e5f196 100644 --- a/cmake/clang-tidy.cmake +++ b/cmake/clang-tidy.cmake @@ -1,6 +1,6 @@ if(NOT CLANG_TIDY_BIN) find_program(CLANG_TIDY_BIN - NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7 + NAMES clang-tidy clang-tidy-20 clang-tidy-19 clang-tidy-18 clang-tidy-17 clang-tidy-16 DOC "Path to clang-tidy executable" ) endif() diff --git a/core/api/service/author/requests/has_key.hpp b/core/api/service/author/requests/has_key.hpp index aeab4da955..fbfb17bdb8 100644 --- a/core/api/service/author/requests/has_key.hpp +++ b/core/api/service/author/requests/has_key.hpp @@ -20,7 +20,7 @@ namespace kagome::api::author::request { public: explicit HasKey(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { OUTCOME_TRY(public_key, common::unhexWith0x(getParam<0>())); diff --git a/core/api/service/author/requests/has_session_keys.hpp b/core/api/service/author/requests/has_session_keys.hpp index 5fee8581b9..ce3f1f3298 100644 --- a/core/api/service/author/requests/has_session_keys.hpp +++ b/core/api/service/author/requests/has_session_keys.hpp @@ -19,7 +19,7 @@ namespace kagome::api::author::request { explicit HasSessionKeys(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { OUTCOME_TRY(keys, common::unhexWith0x(getParam<0>())); diff --git a/core/api/service/author/requests/insert_key.hpp b/core/api/service/author/requests/insert_key.hpp index 43e480276c..f8d1f9ff4e 100644 --- a/core/api/service/author/requests/insert_key.hpp +++ b/core/api/service/author/requests/insert_key.hpp @@ -25,7 +25,7 @@ namespace kagome::api::author::request { public: explicit InsertKey(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { auto &seed_hex = getParam<1>(); diff --git a/core/api/service/author/requests/rotate_keys.hpp b/core/api/service/author/requests/rotate_keys.hpp index 168654d139..e5ee7893a7 100644 --- a/core/api/service/author/requests/rotate_keys.hpp +++ b/core/api/service/author/requests/rotate_keys.hpp @@ -18,7 +18,7 @@ namespace kagome::api::author::request { public: explicit RotateKeys(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { return api_->rotateKeys(); diff --git a/core/api/service/author/requests/submit_and_watch_extrinsic.hpp b/core/api/service/author/requests/submit_and_watch_extrinsic.hpp index 24df06372b..e1846b16a8 100644 --- a/core/api/service/author/requests/submit_and_watch_extrinsic.hpp +++ b/core/api/service/author/requests/submit_and_watch_extrinsic.hpp @@ -9,6 +9,7 @@ #include #include "api/service/author/author_api.hpp" +#include "api/service/base_request.hpp" #include "primitives/extrinsic.hpp" namespace kagome::api::author::request { @@ -19,7 +20,7 @@ namespace kagome::api::author::request { explicit SubmitAndWatchExtrinsic(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { auto ext_hex = getParam<0>(); diff --git a/core/api/service/author/requests/submit_extrinsic.hpp b/core/api/service/author/requests/submit_extrinsic.hpp index d2f9429d98..a073d53689 100644 --- a/core/api/service/author/requests/submit_extrinsic.hpp +++ b/core/api/service/author/requests/submit_extrinsic.hpp @@ -9,6 +9,7 @@ #include #include "api/service/author/author_api.hpp" +#include "api/service/base_request.hpp" #include "primitives/extrinsic.hpp" namespace kagome::api::author::request { @@ -19,7 +20,7 @@ namespace kagome::api::author::request { explicit SubmitExtrinsic(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { auto ext_hex = getParam<0>(); diff --git a/core/api/service/chain/requests/get_block_hash.cpp b/core/api/service/chain/requests/get_block_hash.cpp index 073fa12baa..92b6169160 100644 --- a/core/api/service/chain/requests/get_block_hash.cpp +++ b/core/api/service/chain/requests/get_block_hash.cpp @@ -89,6 +89,6 @@ namespace kagome::api::chain::request { return results; }); - }; + } } // namespace kagome::api::chain::request diff --git a/core/api/service/child_state/requests/get_keys.hpp b/core/api/service/child_state/requests/get_keys.hpp index e3a5b86de1..b2b5940d54 100644 --- a/core/api/service/child_state/requests/get_keys.hpp +++ b/core/api/service/child_state/requests/get_keys.hpp @@ -30,7 +30,7 @@ namespace kagome::api::child_state::request { explicit GetKeys(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } ~GetKeys() = default; outcome::result init(const jsonrpc::Request::Parameters ¶ms); diff --git a/core/api/service/child_state/requests/get_keys_paged.hpp b/core/api/service/child_state/requests/get_keys_paged.hpp index fa70de280a..c9b85f99ba 100644 --- a/core/api/service/child_state/requests/get_keys_paged.hpp +++ b/core/api/service/child_state/requests/get_keys_paged.hpp @@ -30,7 +30,7 @@ namespace kagome::api::child_state::request { explicit GetKeysPaged(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } ~GetKeysPaged() = default; outcome::result init(const jsonrpc::Request::Parameters ¶ms); diff --git a/core/api/service/rpc/requests/methods.hpp b/core/api/service/rpc/requests/methods.hpp index 4b1d80df39..c197c2a1c4 100644 --- a/core/api/service/rpc/requests/methods.hpp +++ b/core/api/service/rpc/requests/methods.hpp @@ -32,7 +32,7 @@ namespace kagome::api::rpc::request { explicit Methods(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } ~Methods() = default; outcome::result init(const jsonrpc::Request::Parameters ¶ms); diff --git a/core/api/service/state/requests/call.cpp b/core/api/service/state/requests/call.cpp index 7581e20a82..80d675ef07 100644 --- a/core/api/service/state/requests/call.cpp +++ b/core/api/service/state/requests/call.cpp @@ -12,7 +12,7 @@ namespace kagome::api::state::request { Call::Call(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result Call::init(const jsonrpc::Request::Parameters ¶ms) { if (params.size() < 2 or params.size() > 3) { diff --git a/core/api/service/state/requests/get_keys_paged.hpp b/core/api/service/state/requests/get_keys_paged.hpp index 1fe24270e3..6b8ae5df07 100644 --- a/core/api/service/state/requests/get_keys_paged.hpp +++ b/core/api/service/state/requests/get_keys_paged.hpp @@ -30,7 +30,7 @@ namespace kagome::api::state::request { explicit GetKeysPaged(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } ~GetKeysPaged() = default; outcome::result init(const jsonrpc::Request::Parameters ¶ms); diff --git a/core/api/service/state/requests/query_storage.hpp b/core/api/service/state/requests/query_storage.hpp index 53095f6674..dd3818d8eb 100644 --- a/core/api/service/state/requests/query_storage.hpp +++ b/core/api/service/state/requests/query_storage.hpp @@ -44,7 +44,7 @@ namespace kagome::api::state::request { explicit QueryStorage(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result> execute() override { diff --git a/core/api/service/system/requests/account_next_index.hpp b/core/api/service/system/requests/account_next_index.hpp index 4d5f6cf90e..a853c6034b 100644 --- a/core/api/service/system/requests/account_next_index.hpp +++ b/core/api/service/system/requests/account_next_index.hpp @@ -17,7 +17,7 @@ namespace kagome::api::system::request { explicit AccountNextIndex(std::shared_ptr api) : api_(std::move(api)) { BOOST_ASSERT(api_); - }; + } outcome::result execute() override { return api_->getNonceFor(getParam<0>()); diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp index e602bb9e8f..0d3946d4d7 100644 --- a/core/application/app_configuration.hpp +++ b/core/application/app_configuration.hpp @@ -211,8 +211,8 @@ namespace kagome::application { * List of telemetry endpoints specified via CLI argument or config file * @return a vector of parsed telemetry endpoints */ - virtual const std::vector - &telemetryEndpoints() const = 0; + virtual const std::vector & + telemetryEndpoints() const = 0; /** * @return enum constant of the chosen sync method diff --git a/core/application/chain_spec.hpp b/core/application/chain_spec.hpp index c5c2b95cae..7536f0d922 100644 --- a/core/application/chain_spec.hpp +++ b/core/application/chain_spec.hpp @@ -38,8 +38,8 @@ namespace kagome::application { virtual const std::vector &bootNodes() const = 0; - virtual const std::vector> - &telemetryEndpoints() const = 0; + virtual const std::vector> & + telemetryEndpoints() const = 0; virtual const std::string &protocolId() const = 0; diff --git a/core/authority_discovery/query/query_impl.cpp b/core/authority_discovery/query/query_impl.cpp index 4472cda14f..ad49eddef5 100644 --- a/core/authority_discovery/query/query_impl.cpp +++ b/core/authority_discovery/query/query_impl.cpp @@ -283,7 +283,7 @@ namespace kagome::authority_discovery { } std::ignore = host_.getPeerRepository().getAddressRepository().addAddresses( - peer.id, peer.addresses, libp2p::peer::ttl::kRecentlyConnected); + peer.id, peer.addresses, libp2p::peer::ttl::kDay); peer_to_auth_cache_.insert_or_assign(peer.id, authority); auth_to_peer_cache_.insert_or_assign( diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 6e92bec81d..603778f2ab 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -425,18 +425,18 @@ namespace kagome::blockchain { std::shared_ptr state_pruner, common::MainThreadPool &main_thread_pool) : block_tree_data_{BlockTreeData{ - .header_repo_ = std::move(header_repo), - .storage_ = std::move(storage), - .state_pruner_ = std::move(state_pruner), - .tree_ = std::make_unique(finalized), - .extrinsic_observer_ = std::move(extrinsic_observer), - .hasher_ = std::move(hasher), - .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo), - .justification_storage_policy_ = - std::move(justification_storage_policy), - .genesis_block_hash_ = {}, - .blocks_pruning_ = {app_config.blocksPruning(), finalized.number}, - }}, + .header_repo_ = std::move(header_repo), + .storage_ = std::move(storage), + .state_pruner_ = std::move(state_pruner), + .tree_ = std::make_unique(finalized), + .extrinsic_observer_ = std::move(extrinsic_observer), + .hasher_ = std::move(hasher), + .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo), + .justification_storage_policy_ = + std::move(justification_storage_policy), + .genesis_block_hash_ = {}, + .blocks_pruning_ = {app_config.blocksPruning(), finalized.number}, + }}, main_pool_handler_{main_thread_pool.handlerStarted()} { block_tree_data_.sharedAccess([&](const BlockTreeData &p) { BOOST_ASSERT(p.header_repo_ != nullptr); diff --git a/core/common/variant_builder.hpp b/core/common/variant_builder.hpp index b56b1d5954..b60f37a320 100644 --- a/core/common/variant_builder.hpp +++ b/core/common/variant_builder.hpp @@ -82,7 +82,7 @@ namespace kagome::common { static_assert(dynamic_variant::is_boost_variant::value); dynamic_variant::functors_vector_builder builder = {&funcs_}; boost::mpl::for_each(builder); - }; + } /** * Initializes the referenced variant with default constructed instance of diff --git a/core/consensus/grandpa/impl/voting_round_impl.hpp b/core/consensus/grandpa/impl/voting_round_impl.hpp index 7e46321aad..2e5c5a6db9 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.hpp +++ b/core/consensus/grandpa/impl/voting_round_impl.hpp @@ -167,7 +167,7 @@ namespace kagome::consensus::grandpa { */ std::shared_ptr getPreviousRound() const override { return previous_round_; - }; + } /** * Removes previous round to limit chain of rounds @@ -226,7 +226,7 @@ namespace kagome::consensus::grandpa { */ const std::optional &finalizedBlock() const override { return finalized_; - }; + } /** * @return state containing round number, last finalized block, votes, and diff --git a/core/consensus/grandpa/types/equivocation_proof.hpp b/core/consensus/grandpa/types/equivocation_proof.hpp index 35ccd8a667..7a46f618a9 100644 --- a/core/consensus/grandpa/types/equivocation_proof.hpp +++ b/core/consensus/grandpa/types/equivocation_proof.hpp @@ -48,7 +48,7 @@ namespace kagome::consensus::grandpa { BOOST_ASSERT((first.is() and second.is()) or (first.is() and second.is())); BOOST_ASSERT(first.id == second.id); - }; + } AuthorityId offender() const { return first.id; diff --git a/core/consensus/timeline/impl/consensus_selector_impl.cpp b/core/consensus/timeline/impl/consensus_selector_impl.cpp index 892b6f1118..6e4d55c97c 100644 --- a/core/consensus/timeline/impl/consensus_selector_impl.cpp +++ b/core/consensus/timeline/impl/consensus_selector_impl.cpp @@ -120,7 +120,7 @@ namespace kagome::consensus { // TODO: Code for trying to select another consensus } BOOST_UNREACHABLE_RETURN({}); - }; + } std::shared_ptr ConsensusSelectorImpl::getFinalityConsensus( @@ -139,6 +139,6 @@ namespace kagome::consensus { // TODO: Code for trying to select another consensus } BOOST_UNREACHABLE_RETURN({}); - }; + } } // namespace kagome::consensus diff --git a/core/consensus/timeline/impl/timeline_impl.cpp b/core/consensus/timeline/impl/timeline_impl.cpp index cbc52a92ca..a2f2f1d150 100644 --- a/core/consensus/timeline/impl/timeline_impl.cpp +++ b/core/consensus/timeline/impl/timeline_impl.cpp @@ -421,7 +421,7 @@ namespace kagome::consensus { } } }); - }; + } bool TimelineImpl::updateSlot(TimePoint now) { best_block_ = block_tree_->bestBlock(); diff --git a/core/crypto/key_store.hpp b/core/crypto/key_store.hpp index 2f638b1b25..d24fc68b99 100644 --- a/core/crypto/key_store.hpp +++ b/core/crypto/key_store.hpp @@ -49,11 +49,11 @@ OUTCOME_HPP_DECLARE_ERROR(kagome::crypto, KeyStoreError); namespace kagome::crypto { template concept Suite = requires() { - typename T::Keypair; - typename T::PrivateKey; - typename T::PublicKey; - typename T::Seed; - }; + typename T::Keypair; + typename T::PrivateKey; + typename T::PublicKey; + typename T::Seed; + }; template class KeySuiteStore { diff --git a/core/crypto/secp256k1/secp256k1_provider_impl.cpp b/core/crypto/secp256k1/secp256k1_provider_impl.cpp index c6fb18a44d..e27d7518d5 100644 --- a/core/crypto/secp256k1/secp256k1_provider_impl.cpp +++ b/core/crypto/secp256k1/secp256k1_provider_impl.cpp @@ -35,7 +35,7 @@ namespace kagome::crypto { } return pubkey_out; - }; + } outcome::result Secp256k1ProviderImpl::recoverPublickeyCompressed( diff --git a/core/dispute_coordinator/impl/batch.hpp b/core/dispute_coordinator/impl/batch.hpp index 2be32366a3..b8170270bc 100644 --- a/core/dispute_coordinator/impl/batch.hpp +++ b/core/dispute_coordinator/impl/batch.hpp @@ -78,7 +78,7 @@ namespace kagome::dispute { TimePoint next_tick_time() const { return next_tick_time_; - }; + } /// Cache of `CandidateHash` (candidate_receipt.hash()). CandidateHash candidate_hash; diff --git a/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp b/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp index b9819f07c7..053ff7812f 100644 --- a/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp +++ b/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp @@ -181,7 +181,7 @@ namespace kagome::dispute { std::make_shared( dispute_thread_pool.io_context()), libp2p::basic::Scheduler::Config{})}, - runtime_info_(std::make_unique(api_, session_keys_)), + runtime_info_(std::make_shared(api_, session_keys_)), batches_(std::make_unique(log_, steady_clock_, hasher_)) { BOOST_ASSERT(session_keys_ != nullptr); BOOST_ASSERT(storage_ != nullptr); @@ -470,6 +470,7 @@ namespace kagome::dispute { std::make_shared(block_header_repository_, hasher_, api_, + runtime_info_, recovery_, pvf_, dispute_thread_handler_, @@ -1205,12 +1206,12 @@ namespace kagome::dispute { auto is_old_concluded_for = intermediate_result.old_state.dispute_status.has_value() ? is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_concluded_for = intermediate_result.new_state.dispute_status.has_value() ? is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_concluded_for = not is_old_concluded_for and is_new_concluded_for; @@ -1218,12 +1219,12 @@ namespace kagome::dispute { auto is_old_concluded_against = intermediate_result.old_state.dispute_status.has_value() ? is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_concluded_against = intermediate_result.new_state.dispute_status.has_value() ? is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_concluded_against = not is_old_concluded_against and is_new_concluded_against; @@ -1234,12 +1235,12 @@ namespace kagome::dispute { auto is_old_confirmed_concluded = intermediate_result.old_state.dispute_status.has_value() ? not is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_confirmed_concluded = intermediate_result.new_state.dispute_status.has_value() ? not is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_confirmed = not is_old_confirmed_concluded and is_new_confirmed_concluded; diff --git a/core/dispute_coordinator/impl/dispute_coordinator_impl.hpp b/core/dispute_coordinator/impl/dispute_coordinator_impl.hpp index fbf5e77d74..bf8f9830c1 100644 --- a/core/dispute_coordinator/impl/dispute_coordinator_impl.hpp +++ b/core/dispute_coordinator/impl/dispute_coordinator_impl.hpp @@ -336,7 +336,7 @@ namespace kagome::dispute { std::shared_ptr scheduler_; - std::unique_ptr runtime_info_; + std::shared_ptr runtime_info_; /// Currently active batches of imports per candidate. std::unique_ptr batches_; diff --git a/core/dispute_coordinator/impl/errors.cpp b/core/dispute_coordinator/impl/errors.cpp index b674f57679..c15e339704 100644 --- a/core/dispute_coordinator/impl/errors.cpp +++ b/core/dispute_coordinator/impl/errors.cpp @@ -45,7 +45,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, DisputeMessageCreationError, e) { return "Invalid statement combination"; } return "unknown error (invalid DisputeMessageCreationError)"; -}; +} OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, DisputeMessageConstructingError, @@ -73,7 +73,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, return "The invalid statement had an invalid validator index"; } return "unknown error (invalid DisputeMessageConstructingError)"; -}; +} OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, DisputeProcessingError, e) { using E = kagome::dispute::DisputeProcessingError; @@ -84,7 +84,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, DisputeProcessingError, e) { return "Authority sent messages at a too high rate"; } return "unknown error (invalid DisputeProcessingError)"; -}; +} OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, BatchError, e) { using E = kagome::dispute::BatchError; @@ -96,7 +96,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, BatchError, e) { return "Received votes from peer have been completely redundant"; } return "unknown error (invalid BatchError)"; -}; +} OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, ApprovalCheckingMultipleCandidatesError, @@ -108,4 +108,4 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::dispute, "candidate"; } return "unknown error (invalid ApprovalCheckingMultipleCandidatesError)"; -}; +} diff --git a/core/dispute_coordinator/participation/impl/participation_impl.cpp b/core/dispute_coordinator/participation/impl/participation_impl.cpp index fe27312b17..78172a22ae 100644 --- a/core/dispute_coordinator/participation/impl/participation_impl.cpp +++ b/core/dispute_coordinator/participation/impl/participation_impl.cpp @@ -9,6 +9,7 @@ #include #include "dispute_coordinator/dispute_coordinator.hpp" +#include "dispute_coordinator/impl/runtime_info.hpp" #include "dispute_coordinator/participation/impl/queues_impl.hpp" #include "dispute_coordinator/participation/queues.hpp" #include "parachain/availability/recovery/recovery.hpp" @@ -23,12 +24,14 @@ namespace kagome::dispute { block_header_repository, std::shared_ptr hasher, std::shared_ptr api, + std::shared_ptr runtime_info, std::shared_ptr recovery, std::shared_ptr pvf, std::shared_ptr dispute_thread_handler, std::weak_ptr dispute_coordinator) : block_header_repository_(std::move(block_header_repository)), api_(std::move(api)), + runtime_info_(std::move(runtime_info)), recovery_(std::move(recovery)), pvf_(std::move(pvf)), dispute_thread_handler_(std::move(dispute_thread_handler)), @@ -37,6 +40,7 @@ namespace kagome::dispute { block_header_repository_, std::move(hasher), api_)) { BOOST_ASSERT(block_header_repository_ != nullptr); BOOST_ASSERT(api_ != nullptr); + BOOST_ASSERT(runtime_info_ != nullptr); BOOST_ASSERT(recovery_ != nullptr); BOOST_ASSERT(pvf_ != nullptr); BOOST_ASSERT(dispute_thread_handler_ != nullptr); @@ -137,8 +141,14 @@ namespace kagome::dispute { void ParticipationImpl::participate(ParticipationRequest request, primitives::BlockHash block_hash) { - auto ctx = std::make_shared( - ParticipationContext{std::move(request), std::move(block_hash)}); + auto info = runtime_info_->get_session_info_by_index( + request.candidate_receipt.descriptor.relay_parent, request.session); + + auto ctx = std::make_shared(ParticipationContext{ + .request = request, + .block_hash = block_hash, + .group_index = info.has_value() ? info.value().validator_info.our_group + : std::nullopt}); participate_stage1( ctx, [request, wp = dispute_coordinator_](ParticipationOutcome res) { @@ -164,7 +174,8 @@ namespace kagome::dispute { recovery_->recover( ctx->request.candidate_receipt, ctx->request.session, - std::nullopt, + ctx->group_index, + std::nullopt, // core_index [wp{weak_from_this()}, ctx, cb = std::move(cb)](auto res_opt) mutable { if (auto self = wp.lock()) { if (not res_opt.has_value()) { diff --git a/core/dispute_coordinator/participation/impl/participation_impl.hpp b/core/dispute_coordinator/participation/impl/participation_impl.hpp index 665ed8c905..0f77893652 100644 --- a/core/dispute_coordinator/participation/impl/participation_impl.hpp +++ b/core/dispute_coordinator/participation/impl/participation_impl.hpp @@ -22,7 +22,8 @@ namespace kagome::parachain { namespace kagome::dispute { class DisputeCoordinator; -} + class RuntimeInfo; +} // namespace kagome::dispute namespace kagome::dispute { @@ -36,6 +37,7 @@ namespace kagome::dispute { block_header_repository, std::shared_ptr hasher, std::shared_ptr api, + std::shared_ptr runtime_info, std::shared_ptr recovery, std::shared_ptr pvf, std::shared_ptr dispute_thread_handler, @@ -66,6 +68,7 @@ namespace kagome::dispute { primitives::BlockHash block_hash; std::optional available_data{}; std::optional validation_code{}; + std::optional group_index{}; }; using ParticipationContextPtr = std::shared_ptr; using ParticipationCallback = std::function; @@ -83,6 +86,7 @@ namespace kagome::dispute { std::shared_ptr block_header_repository_; std::shared_ptr api_; + std::shared_ptr runtime_info_; std::shared_ptr recovery_; std::shared_ptr pvf_; std::shared_ptr dispute_thread_handler_; diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 620489f7fb..4fc5c6d6d5 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -19,8 +19,10 @@ #include "runtime/trie_storage_provider.hpp" #include "storage/predefined_keys.hpp" -#define FFI \ - Ffi ffi { memory_provider_->getCurrentMemory().value().get() } +#define FFI \ + Ffi ffi { \ + memory_provider_->getCurrentMemory().value().get() \ + } namespace kagome::host_api { /** diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 5508a35f46..c64e75fe09 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -237,7 +237,7 @@ namespace { template auto useConfig(C c) { - return boost::di::bind>().template to( + return boost::di::bind>().to( std::move(c))[boost::di::override]; } @@ -883,6 +883,11 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), // user-defined overrides... std::forward(args)...); diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index 1c0b1c7d16..347febe96f 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -46,24 +46,25 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, PeerManagerImpl::Error, e) { namespace { template - bool openOutgoing(std::shared_ptr &se, - const std::shared_ptr

&protocol, - const kagome::network::PeerManager::PeerInfo &pi, - F &&func) { - BOOST_ASSERT(se); + bool openOutgoing( + std::shared_ptr &stream_engine, + const std::shared_ptr

&protocol, + const kagome::network::PeerManager::PeerInfo &peer_info, + F &&func) { // NOLINT(cppcoreguidelines-missing-std-forward) + BOOST_ASSERT(stream_engine); BOOST_ASSERT(protocol); - if (se->reserveOutgoing(pi.id, protocol)) { + if (stream_engine->reserveOutgoing(peer_info.id, protocol)) { protocol->newOutgoingStream( - pi, - [pid{pi.id}, + peer_info.id, + [peer_id{peer_info.id}, wptr_proto{std::weak_ptr

{protocol}}, - wptr_se{std::weak_ptr{se}}, + wptr_se{std::weak_ptr{stream_engine}}, func{std::forward(func)}](auto &&stream) mutable { - auto se = wptr_se.lock(); + auto stream_engine = wptr_se.lock(); auto proto = wptr_proto.lock(); - if (se && proto) { - se->dropReserveOutgoing(pid, proto); + if (stream_engine && proto) { + stream_engine->dropReserveOutgoing(peer_id, proto); } std::forward(func)(std::forward(stream)); }); diff --git a/core/network/impl/protocols/beefy_justification_protocol.cpp b/core/network/impl/protocols/beefy_justification_protocol.cpp index 91cff9c3dc..3d25e1b11d 100644 --- a/core/network/impl/protocols/beefy_justification_protocol.cpp +++ b/core/network/impl/protocols/beefy_justification_protocol.cpp @@ -15,12 +15,13 @@ #include "network/peer_manager.hpp" namespace kagome::network { + BeefyJustificationProtocol::BeefyJustificationProtocol(libp2p::Host &host, const blockchain::GenesisBlockHash &genesis, common::MainThreadPool &main_thread_pool, std::shared_ptr peer_manager, std::shared_ptr beefy) - : RequestResponseProtocolType{ + : RequestResponseProtocolImpl{ kName, host, make_protocols(kBeefyJustificationProtocol, genesis), diff --git a/core/network/impl/protocols/beefy_justification_protocol.hpp b/core/network/impl/protocols/beefy_justification_protocol.hpp index 3519c154d1..3291e3b0a1 100644 --- a/core/network/impl/protocols/beefy_justification_protocol.hpp +++ b/core/network/impl/protocols/beefy_justification_protocol.hpp @@ -34,9 +34,9 @@ namespace kagome::network { class BeefyJustificationProtocol : public consensus::beefy::FetchJustification, - public RequestResponseProtocol { + public RequestResponseProtocolImpl { static constexpr auto kName = "BeefyJustificationProtocol"; public: diff --git a/core/network/impl/protocols/beefy_protocol_impl.cpp b/core/network/impl/protocols/beefy_protocol_impl.cpp index e6b687f635..8e156baf26 100644 --- a/core/network/impl/protocols/beefy_protocol_impl.cpp +++ b/core/network/impl/protocols/beefy_protocol_impl.cpp @@ -60,7 +60,7 @@ namespace kagome::network { } void BeefyProtocolImpl::newOutgoingStream( - const PeerInfo &peer, + const PeerId &peer_id, std::function>)> &&cb) { auto on_handshake = [cb = std::move(cb)]( @@ -76,7 +76,7 @@ namespace kagome::network { cb(std::move(stream)); }; notifications::connectAndHandshake( - weak_from_this(), base_, peer, roles_, std::move(on_handshake)); + weak_from_this(), base_, peer_id, roles_, std::move(on_handshake)); } void BeefyProtocolImpl::broadcast( diff --git a/core/network/impl/protocols/beefy_protocol_impl.hpp b/core/network/impl/protocols/beefy_protocol_impl.hpp index c54351260a..968d799b6c 100644 --- a/core/network/impl/protocols/beefy_protocol_impl.hpp +++ b/core/network/impl/protocols/beefy_protocol_impl.hpp @@ -39,7 +39,7 @@ namespace kagome::network { const std::string &protocolName() const override; void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/block_announce_protocol.cpp b/core/network/impl/protocols/block_announce_protocol.cpp index bea92e0186..aec7973556 100644 --- a/core/network/impl/protocols/block_announce_protocol.cpp +++ b/core/network/impl/protocols/block_announce_protocol.cpp @@ -110,14 +110,14 @@ namespace kagome::network { } void BlockAnnounceProtocol::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { SL_DEBUG(base_.logger(), "Connect for {} stream with {}", protocolName(), - peer_info.id); + peer_id); - auto on_handshake = [peer_id = peer_info.id, cb = std::move(cb)]( + auto on_handshake = [peer_id, cb = std::move(cb)]( std::shared_ptr self, outcome::result> r) mutable { @@ -135,7 +135,7 @@ namespace kagome::network { }; notifications::connectAndHandshake(weak_from_this(), base_, - peer_info, + peer_id, createHandshake(), std::move(on_handshake)); } diff --git a/core/network/impl/protocols/block_announce_protocol.hpp b/core/network/impl/protocols/block_announce_protocol.hpp index 905bcd554f..1e621ae58f 100644 --- a/core/network/impl/protocols/block_announce_protocol.hpp +++ b/core/network/impl/protocols/block_announce_protocol.hpp @@ -59,7 +59,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/fetch_attested_candidate.hpp b/core/network/impl/protocols/fetch_attested_candidate.hpp index 4812c0a61a..8475841d52 100644 --- a/core/network/impl/protocols/fetch_attested_candidate.hpp +++ b/core/network/impl/protocols/fetch_attested_candidate.hpp @@ -24,21 +24,18 @@ namespace kagome::network { class FetchAttestedCandidateProtocol final - : public RequestResponseProtocol, + : public RequestResponseProtocolImpl, NonCopyable, NonMovable { public: - FetchAttestedCandidateProtocol() = delete; - ~FetchAttestedCandidateProtocol() override = default; - FetchAttestedCandidateProtocol( libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr pp) - : RequestResponseProtocol< + : RequestResponseProtocolImpl< vstaging::AttestedCandidateRequest, vstaging::AttestedCandidateResponse, ScaleMessageReadWriter>{kFetchAttestedCandidateProtocolName, diff --git a/core/network/impl/protocols/grandpa_protocol.cpp b/core/network/impl/protocols/grandpa_protocol.cpp index 593b698c38..2d25cd9e19 100644 --- a/core/network/impl/protocols/grandpa_protocol.cpp +++ b/core/network/impl/protocols/grandpa_protocol.cpp @@ -75,7 +75,7 @@ namespace kagome::network { } void GrandpaProtocol::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { auto on_handshake = [cb = std::move(cb)]( @@ -96,7 +96,7 @@ namespace kagome::network { cb(std::move(stream)); }; notifications::connectAndHandshake( - weak_from_this(), base_, peer_info, roles_, std::move(on_handshake)); + weak_from_this(), base_, peer_id, roles_, std::move(on_handshake)); } void GrandpaProtocol::onMessage(const PeerId &peer_id, diff --git a/core/network/impl/protocols/grandpa_protocol.hpp b/core/network/impl/protocols/grandpa_protocol.hpp index 4f63889859..f0036a019f 100644 --- a/core/network/impl/protocols/grandpa_protocol.hpp +++ b/core/network/impl/protocols/grandpa_protocol.hpp @@ -60,7 +60,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/light.cpp b/core/network/impl/protocols/light.cpp index 0aeca423e8..2c8fea8681 100644 --- a/core/network/impl/protocols/light.cpp +++ b/core/network/impl/protocols/light.cpp @@ -22,7 +22,7 @@ namespace kagome::network { std::shared_ptr storage, std::shared_ptr module_repo, std::shared_ptr executor) - : RequestResponseProtocolType{ + : RequestResponseProtocolImpl{ kName, host, make_protocols(kLightProtocol, genesis, chain_spec), diff --git a/core/network/impl/protocols/light.hpp b/core/network/impl/protocols/light.hpp index cd2ab4b3e3..3620c9ae37 100644 --- a/core/network/impl/protocols/light.hpp +++ b/core/network/impl/protocols/light.hpp @@ -37,9 +37,9 @@ namespace kagome::network { * https://github.com/paritytech/substrate/tree/master/client/network/light */ class LightProtocol - : public RequestResponseProtocol { + : public RequestResponseProtocolImpl { static constexpr auto kName = "LightProtocol"; public: diff --git a/core/network/impl/protocols/parachain_protocol.hpp b/core/network/impl/protocols/parachain_protocol.hpp index f9c4faf956..292589db35 100644 --- a/core/network/impl/protocols/parachain_protocol.hpp +++ b/core/network/impl/protocols/parachain_protocol.hpp @@ -135,13 +135,13 @@ namespace kagome::network { * established. */ void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override { SL_DEBUG(base_.logger(), "Connect for {} stream with {}", protocolName(), - peer_info.id); + peer_id); // This lambda function is called when a handshake is received. auto on_handshake = @@ -158,7 +158,7 @@ namespace kagome::network { }; notifications::connectAndHandshake(this->weak_from_this(), base_, - peer_info, + peer_id, roles_, std::move(on_handshake)); } diff --git a/core/network/impl/protocols/propagate_transactions_protocol.cpp b/core/network/impl/protocols/propagate_transactions_protocol.cpp index 6b0e99a488..e164be5975 100644 --- a/core/network/impl/protocols/propagate_transactions_protocol.cpp +++ b/core/network/impl/protocols/propagate_transactions_protocol.cpp @@ -116,7 +116,7 @@ namespace kagome::network { } void PropagateTransactionsProtocol::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { auto on_handshake = [cb = std::move(cb)]( @@ -132,7 +132,7 @@ namespace kagome::network { cb(std::move(stream)); }; notifications::connectAndHandshake( - weak_from_this(), base_, peer_info, roles_, std::move(on_handshake)); + weak_from_this(), base_, peer_id, roles_, std::move(on_handshake)); } void PropagateTransactionsProtocol::propagateTransactions( diff --git a/core/network/impl/protocols/propagate_transactions_protocol.hpp b/core/network/impl/protocols/propagate_transactions_protocol.hpp index 4ecf717c27..95dc99aafa 100644 --- a/core/network/impl/protocols/propagate_transactions_protocol.hpp +++ b/core/network/impl/protocols/propagate_transactions_protocol.hpp @@ -78,7 +78,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/protocol_fetch_available_data.hpp b/core/network/impl/protocols/protocol_fetch_available_data.hpp index 3c2b91f113..cee4c17b5e 100644 --- a/core/network/impl/protocols/protocol_fetch_available_data.hpp +++ b/core/network/impl/protocols/protocol_fetch_available_data.hpp @@ -16,19 +16,25 @@ #include "parachain/backing/store.hpp" namespace kagome::network { - class FetchAvailableDataProtocol final - : public RequestResponseProtocol { + + class FetchAvailableDataProtocol + : virtual public RequestResponseProtocol {}; + + class FetchAvailableDataProtocolImpl final + : public FetchAvailableDataProtocol, + public RequestResponseProtocolImpl { public: static constexpr const char *kName = "FetchAvailableDataProtocol"; - FetchAvailableDataProtocol( + FetchAvailableDataProtocolImpl( libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr av_store) - : RequestResponseProtocol< + : RequestResponseProtocolImpl< FetchAvailableDataRequest, FetchAvailableDataResponse, ScaleMessageReadWriter>{kName, @@ -55,9 +61,9 @@ namespace kagome::network { }; class StatementFetchingProtocol final - : public RequestResponseProtocol { + : public RequestResponseProtocolImpl { public: static constexpr const char *kName = "FetchStatementProtocol"; @@ -66,7 +72,7 @@ namespace kagome::network { const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr backing_store) - : RequestResponseProtocol< + : RequestResponseProtocolImpl< FetchStatementRequest, FetchStatementResponse, ScaleMessageReadWriter>{kName, diff --git a/core/network/impl/protocols/protocol_fetch_chunk.hpp b/core/network/impl/protocols/protocol_fetch_chunk.hpp index 062ae1bf38..c1a611636e 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk.hpp @@ -25,21 +25,24 @@ namespace kagome::network { struct ReqPovProtocolImpl; - class FetchChunkProtocol final - : public RequestResponseProtocol, + class FetchChunkProtocol + : virtual public RequestResponseProtocol {}; + + class FetchChunkProtocolImpl final + : public FetchChunkProtocol, + public RequestResponseProtocolImpl, NonCopyable, NonMovable { public: - FetchChunkProtocol() = delete; - ~FetchChunkProtocol() override = default; - - FetchChunkProtocol(libp2p::Host &host, - const application::ChainSpec & /*chain_spec*/, - const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr pp) - : RequestResponseProtocol< + FetchChunkProtocolImpl( + libp2p::Host &host, + const application::ChainSpec & /*chain_spec*/, + const blockchain::GenesisBlockHash &genesis_hash, + std::shared_ptr pp) + : RequestResponseProtocolImpl< FetchChunkRequest, FetchChunkResponse, ScaleMessageReadWriter>{kFetchChunkProtocolName, diff --git a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp index 7e5f71c4b1..ada77b6491 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp @@ -23,28 +23,28 @@ namespace kagome::network { - struct ReqPovProtocolImpl; + class FetchChunkProtocolObsolete + : virtual public RequestResponseProtocol {}; /// Implementation of first implementation of /// fetching chunk protocol aka 'req_chunk/1' /// /// In response index of systematic chunk is corresponding validator index. - class FetchChunkProtocolObsolete final - : public RequestResponseProtocol, + class FetchChunkProtocolObsoleteImpl final + : public FetchChunkProtocolObsolete, + public RequestResponseProtocolImpl, NonCopyable, NonMovable { public: - FetchChunkProtocolObsolete() = delete; - ~FetchChunkProtocolObsolete() override = default; - - FetchChunkProtocolObsolete( + FetchChunkProtocolObsoleteImpl( libp2p::Host &host, const application::ChainSpec & /*chain_spec*/, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr pp) - : RequestResponseProtocol< + : RequestResponseProtocolImpl< FetchChunkRequest, FetchChunkResponseObsolete, ScaleMessageReadWriter>{kFetchChunkProtocolName, diff --git a/core/network/impl/protocols/protocol_req_collation.cpp b/core/network/impl/protocols/protocol_req_collation.cpp index 4a0449b936..b8f5458a53 100644 --- a/core/network/impl/protocols/protocol_req_collation.cpp +++ b/core/network/impl/protocols/protocol_req_collation.cpp @@ -15,14 +15,14 @@ namespace kagome::network { template struct ReqCollationProtocolImpl - : RequestResponseProtocol, - std::decay_t, - ScaleMessageReadWriter>, + : RequestResponseProtocolImpl, + std::decay_t, + ScaleMessageReadWriter>, NonCopyable, NonMovable { - using Base = RequestResponseProtocol, - std::decay_t, - ScaleMessageReadWriter>; + using Base = RequestResponseProtocolImpl, + std::decay_t, + ScaleMessageReadWriter>; ReqCollationProtocolImpl(libp2p::Host &host, const libp2p::peer::ProtocolName &protoname, @@ -92,7 +92,7 @@ namespace kagome::network { void ReqCollationProtocol::onIncomingStream(std::shared_ptr stream) {} void ReqCollationProtocol::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { BOOST_ASSERT_MSG(false, "Must not be called!"); } diff --git a/core/network/impl/protocols/protocol_req_collation.hpp b/core/network/impl/protocols/protocol_req_collation.hpp index 9c7b0b7872..d10dbe94af 100644 --- a/core/network/impl/protocols/protocol_req_collation.hpp +++ b/core/network/impl/protocols/protocol_req_collation.hpp @@ -50,7 +50,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/protocol_req_pov.cpp b/core/network/impl/protocols/protocol_req_pov.cpp index bd8a0eaa05..7702ee9520 100644 --- a/core/network/impl/protocols/protocol_req_pov.cpp +++ b/core/network/impl/protocols/protocol_req_pov.cpp @@ -13,25 +13,26 @@ namespace kagome::network { - struct ReqPovProtocolImpl : RequestResponseProtocol, - NonCopyable, - NonMovable { + struct ReqPovProtocolImpl + : RequestResponseProtocolImpl, + NonCopyable, + NonMovable { ReqPovProtocolImpl(libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr observer) - : RequestResponseProtocol< - RequestPov, - ResponsePov, - ScaleMessageReadWriter>{kReqPovProtocolName, - host, - make_protocols(kReqPovProtocol, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger(kReqPovProtocolName, - "req_pov_protocol")}, + : RequestResponseProtocolImpl< + RequestPov, + ResponsePov, + ScaleMessageReadWriter>{kReqPovProtocolName, + host, + make_protocols(kReqPovProtocol, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger(kReqPovProtocolName, + "req_pov_protocol")}, observer_{std::move(observer)} {} protected: @@ -71,7 +72,7 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr observer) : impl_{std::make_shared( - host, chain_spec, genesis_hash, std::move(observer))} {} + host, chain_spec, genesis_hash, std::move(observer))} {} const Protocol &ReqPovProtocol::protocolName() const { BOOST_ASSERT(impl_ && !!"ReqPovProtocolImpl must be initialized!"); @@ -88,18 +89,17 @@ namespace kagome::network { } void ReqPovProtocol::newOutgoingStream( - const PeerInfo &, + const PeerId &, std::function>)> &&) { BOOST_ASSERT(!"Must not be called!"); } void ReqPovProtocol::request( - const PeerInfo &peer_info, + const PeerId &peer_id, RequestPov request, std::function)> &&response_handler) { BOOST_ASSERT(impl_ && !!"ReqPovProtocolImpl must be initialized!"); - return impl_->doRequest( - peer_info, std::move(request), std::move(response_handler)); + return impl_->doRequest(peer_id, request, std::move(response_handler)); } } // namespace kagome::network diff --git a/core/network/impl/protocols/protocol_req_pov.hpp b/core/network/impl/protocols/protocol_req_pov.hpp index fdaa7330a8..ceba2f6198 100644 --- a/core/network/impl/protocols/protocol_req_pov.hpp +++ b/core/network/impl/protocols/protocol_req_pov.hpp @@ -45,11 +45,11 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; - void request(const PeerInfo &peer_info, + void request(const PeerId &peer_id, RequestPov, std::function)> &&response_handler) override; diff --git a/core/network/impl/protocols/request_response_protocol.hpp b/core/network/impl/protocols/request_response_protocol.hpp index a56c80e244..c7a2e5c3f1 100644 --- a/core/network/impl/protocols/request_response_protocol.hpp +++ b/core/network/impl/protocols/request_response_protocol.hpp @@ -13,18 +13,28 @@ namespace kagome::network { + template + struct RequestResponseProtocol : virtual public ProtocolBase { + using RequestType = Request; + using ResponseType = Response; + + virtual void doRequest(const PeerId &peer_id, + RequestType request, + std::function)> + &&response_handler) = 0; + }; + template - struct RequestResponseProtocol - : ProtocolBase, + struct RequestResponseProtocolImpl + : virtual protected ProtocolBase, + virtual public RequestResponseProtocol, std::enable_shared_from_this< - RequestResponseProtocol> { - using RequestResponseProtocolType = - RequestResponseProtocol; + RequestResponseProtocolImpl> { using RequestType = Request; using ResponseType = Response; using ReadWriterType = ReadWriter; - RequestResponseProtocol( + RequestResponseProtocolImpl( Protocol name, libp2p::Host &host, Protocols protocols, @@ -32,7 +42,6 @@ namespace kagome::network { std::chrono::milliseconds timeout = std::chrono::seconds(1)) : base_(std::move(name), host, std::move(protocols), std::move(logger)), timeout_(std::move(timeout)) {} - virtual ~RequestResponseProtocol() {} bool start() override { return base_.start(this->weak_from_this()); @@ -42,20 +51,13 @@ namespace kagome::network { return base_.protocolName(); } - void doRequest( - const PeerId &peer_id, - RequestType request, - std::function)> &&response_handler) { - doRequest({peer_id, {}}, std::move(request), std::move(response_handler)); - } - - void doRequest( - const PeerInfo &peer_info, - RequestType request, - std::function)> &&response_handler) { + void doRequest(const PeerId &peer_id, + Request request, + std::function)> + &&response_handler) override { onTxRequest(request); newOutgoingStream( - peer_info, + peer_id, [wptr{this->weak_from_this()}, request{std::move(request)}, response_handler{std::move(response_handler)}](auto &&res) mutable { @@ -106,20 +108,27 @@ namespace kagome::network { } void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override { SL_TRACE(base_.logger(), - "Connect for {} stream with {}", + "New outgoing {} stream with {}", protocolName(), - peer_info.id); + peer_id); + + auto addresses_res = + base_.host().getPeerRepository().getAddressRepository().getAddresses( + peer_id); + if (not addresses_res.has_value()) { + cb(addresses_res.as_failure()); + return; + } base_.host().newStream( - peer_info, + PeerInfo{peer_id, std::move(addresses_res.value())}, base_.protocolIds(), - [wptr{this->weak_from_this()}, - peer_id{peer_info.id}, - cb{std::move(cb)}](auto &&stream_and_proto) mutable { + [wptr{this->weak_from_this()}, peer_id, cb{std::move(cb)}]( + auto &&stream_and_proto) mutable { if (!stream_and_proto.has_value()) { cb(stream_and_proto.as_failure()); return; diff --git a/core/network/impl/protocols/send_dispute_protocol.hpp b/core/network/impl/protocols/send_dispute_protocol.hpp index e7f6847b17..20395a09c9 100644 --- a/core/network/impl/protocols/send_dispute_protocol.hpp +++ b/core/network/impl/protocols/send_dispute_protocol.hpp @@ -27,25 +27,28 @@ namespace kagome::network { - struct ReqPovProtocolImpl; - - class SendDisputeProtocol final - : public RequestResponseProtocol, - ScaleMessageReadWriter>, + using DisputeRequest = DisputeMessage; + using DisputeResponse = boost::variant; + + class SendDisputeProtocol + : virtual public RequestResponseProtocol {}; + + class SendDisputeProtocolImpl final + : public SendDisputeProtocol, + public RequestResponseProtocolImpl, NonCopyable, NonMovable { public: - SendDisputeProtocol() = delete; - ~SendDisputeProtocol() override = default; - - SendDisputeProtocol(libp2p::Host &host, - const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr - dispute_request_observer) - : RequestResponseProtocol< - DisputeMessage, - boost::variant, + SendDisputeProtocolImpl(libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis_hash, + std::shared_ptr + dispute_request_observer) + : RequestResponseProtocolImpl< + DisputeRequest, + DisputeResponse, ScaleMessageReadWriter>{kSendDisputeProtocolName, host, make_protocols(kSendDisputeProtocol, @@ -73,8 +76,8 @@ namespace kagome::network { peer_id, std::move(request), [wp{weak_from_this()}, - base = &SendDisputeProtocol::base, - write = &SendDisputeProtocol::writeResponse, + base = &SendDisputeProtocolImpl::base, + write = &SendDisputeProtocolImpl::writeResponse, stream = std::move(stream)](outcome::result res) mutable { BOOST_ASSERT(stream); diff --git a/core/network/impl/protocols/state_protocol_impl.cpp b/core/network/impl/protocols/state_protocol_impl.cpp index eecfcb8c32..4d6704ffcb 100644 --- a/core/network/impl/protocols/state_protocol_impl.cpp +++ b/core/network/impl/protocols/state_protocol_impl.cpp @@ -38,17 +38,17 @@ namespace kagome::network { } void StateProtocolImpl::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { SL_DEBUG(base_.logger(), "Connect for {} stream with {}", protocolName(), - peer_info.id); + peer_id); base_.host().newStream( - peer_info.id, + peer_id, base_.protocolIds(), - [wp{weak_from_this()}, peer_id = peer_info.id, cb = std::move(cb)]( + [wp{weak_from_this()}, peer_id, cb = std::move(cb)]( auto &&stream_res) mutable { auto self = wp.lock(); if (not self) { @@ -150,16 +150,8 @@ namespace kagome::network { const PeerId &peer_id, StateRequest state_request, std::function)> &&response_handler) { - auto addresses_res = - base_.host().getPeerRepository().getAddressRepository().getAddresses( - peer_id); - if (not addresses_res.has_value()) { - response_handler(addresses_res.as_failure()); - return; - } - newOutgoingStream( - {peer_id, std::move(addresses_res.value())}, + peer_id, [wp{weak_from_this()}, response_handler = std::move(response_handler), state_request = std::move(state_request)](auto &&stream_res) mutable { diff --git a/core/network/impl/protocols/state_protocol_impl.hpp b/core/network/impl/protocols/state_protocol_impl.hpp index 41d41d47a4..2c1c4fc595 100644 --- a/core/network/impl/protocols/state_protocol_impl.hpp +++ b/core/network/impl/protocols/state_protocol_impl.hpp @@ -49,7 +49,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/protocols/sync_protocol_impl.cpp b/core/network/impl/protocols/sync_protocol_impl.cpp index 258d60d137..5d1b76a29d 100644 --- a/core/network/impl/protocols/sync_protocol_impl.cpp +++ b/core/network/impl/protocols/sync_protocol_impl.cpp @@ -154,17 +154,25 @@ namespace kagome::network { } void SyncProtocolImpl::newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) { - SL_DEBUG(base_.logger(), - "Connect for {} stream with {}", + SL_TRACE(base_.logger(), + "New outgoing {} stream with {}", protocolName(), - peer_info.id); + peer_id); + + auto addresses_res = + base_.host().getPeerRepository().getAddressRepository().getAddresses( + peer_id); + if (not addresses_res.has_value()) { + cb(addresses_res.as_failure()); + return; + } base_.host().newStream( - peer_info.id, + PeerInfo{peer_id, std::move(addresses_res.value())}, base_.protocolIds(), - [wp{weak_from_this()}, peer_id = peer_info.id, cb = std::move(cb)]( + [wp{weak_from_this()}, peer_id, cb = std::move(cb)]( auto &&stream_res) mutable { network::streamReadBuffer(stream_res); auto self = wp.lock(); @@ -427,15 +435,7 @@ namespace kagome::network { const PeerId &peer_id, BlocksRequest block_request, std::function)> &&response_handler) { - auto addresses_res = - base_.host().getPeerRepository().getAddressRepository().getAddresses( - peer_id); - if (not addresses_res.has_value()) { - response_handler(addresses_res.as_failure()); - return; - } - - if (base_.logger()->level() >= log::Level::DEBUG) { + [[unlikely]] if (base_.logger()->level() >= log::Level::DEBUG) { std::string logmsg = "Requesting blocks: fields="; if (has(block_request.fields, BlockAttribute::HEADER)) { @@ -469,7 +469,7 @@ namespace kagome::network { } newOutgoingStream( - {peer_id, addresses_res.value()}, + peer_id, [wp{weak_from_this()}, response_handler = std::move(response_handler), block_request = std::move(block_request)](auto &&stream_res) mutable { diff --git a/core/network/impl/protocols/sync_protocol_impl.hpp b/core/network/impl/protocols/sync_protocol_impl.hpp index 831fa96b94..ebb6ae8ac7 100644 --- a/core/network/impl/protocols/sync_protocol_impl.hpp +++ b/core/network/impl/protocols/sync_protocol_impl.hpp @@ -124,7 +124,7 @@ namespace kagome::network { void onIncomingStream(std::shared_ptr stream) override; void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) override; diff --git a/core/network/impl/router_libp2p.cpp b/core/network/impl/router_libp2p.cpp index fb7eed57e3..08f6c30a3b 100644 --- a/core/network/impl/router_libp2p.cpp +++ b/core/network/impl/router_libp2p.cpp @@ -226,6 +226,10 @@ namespace kagome::network { return grandpa_protocol_.get(); } + std::shared_ptr RouterLibp2p::getWarpProtocol() const { + return warp_protocol_.get(); + } + std::shared_ptr RouterLibp2p::getSyncProtocol() const { return sync_protocol_.get(); } diff --git a/core/network/impl/router_libp2p.hpp b/core/network/impl/router_libp2p.hpp index baddb46541..8721848eaf 100644 --- a/core/network/impl/router_libp2p.hpp +++ b/core/network/impl/router_libp2p.hpp @@ -90,6 +90,7 @@ namespace kagome::network { const override; std::shared_ptr getGrandpaProtocol() const override; + std::shared_ptr getWarpProtocol() const override; std::shared_ptr getSyncProtocol() const override; std::shared_ptr getStateProtocol() const override; diff --git a/core/network/impl/stream_engine.cpp b/core/network/impl/stream_engine.cpp index 03d1616e8f..96723b0a4a 100644 --- a/core/network/impl/stream_engine.cpp +++ b/core/network/impl/stream_engine.cpp @@ -176,7 +176,7 @@ namespace kagome::network { ProtocolDescr &descr) { if (descr.tryReserveOutgoing()) { protocol->newOutgoingStream( - PeerInfo{peer_id, {}}, + peer_id, [wp(weak_from_this()), protocol, peer_id](auto &&stream_res) mutable { auto self = wp.lock(); if (not self) { diff --git a/core/network/notifications/connect_and_handshake.hpp b/core/network/notifications/connect_and_handshake.hpp index c50d9a47ad..37e3077894 100644 --- a/core/network/notifications/connect_and_handshake.hpp +++ b/core/network/notifications/connect_and_handshake.hpp @@ -21,7 +21,7 @@ namespace kagome::network::notifications { template void connectAndHandshake(std::weak_ptr weak, ProtocolBaseImpl &base, - const PeerInfo &peer, + const PeerId &peer_id, Handshake handshake, OnHandshake on_handshake) { auto cb = [weak = std::move(weak), @@ -64,6 +64,16 @@ namespace kagome::network::notifications { std::move(stream), std::move(frame_stream), handshake, std::move(cb)); }; - base.host().newStream(peer, base.protocolIds(), std::move(cb)); + auto addresses_res = + base.host().getPeerRepository().getAddressRepository().getAddresses( + peer_id); + if (not addresses_res.has_value()) { + cb(addresses_res.as_failure()); + return; + } + + base.host().newStream(PeerInfo{peer_id, std::move(addresses_res.value())}, + base.protocolIds(), + std::move(cb)); } } // namespace kagome::network::notifications diff --git a/core/network/notifications/handshake.hpp b/core/network/notifications/handshake.hpp index 2fa4837649..772f8f4ae6 100644 --- a/core/network/notifications/handshake.hpp +++ b/core/network/notifications/handshake.hpp @@ -6,6 +6,8 @@ #pragma once +#include "common/buffer_view.hpp" + #include #include #include diff --git a/core/network/protocol_base.hpp b/core/network/protocol_base.hpp index 3e1ca4efb1..55ddcefd3d 100644 --- a/core/network/protocol_base.hpp +++ b/core/network/protocol_base.hpp @@ -38,7 +38,7 @@ namespace kagome::network { virtual void onIncomingStream(std::shared_ptr stream) = 0; virtual void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&cb) = 0; }; diff --git a/core/network/protocols/req_pov_protocol.hpp b/core/network/protocols/req_pov_protocol.hpp index c56293a210..74e3e003d6 100644 --- a/core/network/protocols/req_pov_protocol.hpp +++ b/core/network/protocols/req_pov_protocol.hpp @@ -17,7 +17,7 @@ namespace kagome::network { class IReqPovProtocol : public ProtocolBase { public: - virtual void request(const PeerInfo &peer_info, + virtual void request(const PeerId &peer_id, RequestPov, std::function)> &&response_handler) = 0; diff --git a/core/network/req_pov_observer.hpp b/core/network/req_pov_observer.hpp index 93f3e6311c..32effc7aac 100644 --- a/core/network/req_pov_observer.hpp +++ b/core/network/req_pov_observer.hpp @@ -6,6 +6,7 @@ #pragma once +#include "network/types/collator_messages.hpp" #include "primitives/common.hpp" namespace kagome::network { diff --git a/core/network/router.hpp b/core/network/router.hpp index c0bbe080d7..56a8ee9ff0 100644 --- a/core/network/router.hpp +++ b/core/network/router.hpp @@ -23,6 +23,7 @@ namespace kagome::network { class FetchAvailableDataProtocol; class StatementFetchingProtocol; class PropagateTransactionsProtocol; + class WarpProtocol; class StateProtocol; class SyncProtocol; class GrandpaProtocol; @@ -31,6 +32,7 @@ namespace kagome::network { class CollationProtocolVStaging; class ValidationProtocolVStaging; class FetchAttestedCandidateProtocol; + using Ping = libp2p::protocol::Ping; } // namespace kagome::network namespace kagome::network { @@ -67,6 +69,7 @@ namespace kagome::network { getFetchStatementProtocol() const = 0; virtual std::shared_ptr getPropagateTransactionsProtocol() const = 0; + virtual std::shared_ptr getWarpProtocol() const = 0; virtual std::shared_ptr getStateProtocol() const = 0; virtual std::shared_ptr getSyncProtocol() const = 0; virtual std::shared_ptr getGrandpaProtocol() const = 0; diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index b0e056bece..bda985633b 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -69,7 +69,7 @@ namespace kagome::network { boost::hash_combine(result, std::hash()(voter_set_id)); return result; - }; + } }; struct CatchUpResponse { diff --git a/core/network/warp/protocol.hpp b/core/network/warp/protocol.hpp index b6e164bf78..b8fba8b027 100644 --- a/core/network/warp/protocol.hpp +++ b/core/network/warp/protocol.hpp @@ -12,17 +12,26 @@ #include "network/warp/cache.hpp" namespace kagome::network { - class WarpProtocol : public RequestResponseProtocol { + + using WarpRequest = primitives::BlockHash; + using WarpResponse = WarpSyncProof; + + class WarpProtocol + : virtual public RequestResponseProtocol {}; + + class WarpProtocolImpl + : public WarpProtocol, + public RequestResponseProtocolImpl { static constexpr auto kName = "WarpProtocol"; public: - WarpProtocol(libp2p::Host &host, - const application::ChainSpec &chain_spec, - const blockchain::GenesisBlockHash &genesis, - std::shared_ptr cache) - : RequestResponseProtocolType( + WarpProtocolImpl(libp2p::Host &host, + const application::ChainSpec &chain_spec, + const blockchain::GenesisBlockHash &genesis, + std::shared_ptr cache) + : RequestResponseProtocolImpl( kName, host, make_protocols(kWarpProtocol, genesis, chain_spec), diff --git a/core/parachain/approval/approval_distribution.cpp b/core/parachain/approval/approval_distribution.cpp index 116777d180..70215e98f3 100644 --- a/core/parachain/approval/approval_distribution.cpp +++ b/core/parachain/approval/approval_distribution.cpp @@ -619,7 +619,8 @@ namespace kagome::parachain { metrics_registry_->registerCounterFamily( kMetricNoShowsTotal, - "Number of assignments which became no-shows in the approval voting subsystem"); + "Number of assignments which became no-shows in the approval voting " + "subsystem"); metric_no_shows_total_ = metrics_registry_->registerCounterMetric(kMetricNoShowsTotal); } @@ -1614,6 +1615,7 @@ namespace kagome::parachain { const HashedCandidateReceipt &hashed_candidate, ValidatorIndex validator_index, Hash block_hash, + std::optional core, GroupIndex backing_group) { auto on_recover_complete = [wself{weak_from_this()}, @@ -1725,6 +1727,7 @@ namespace kagome::parachain { recovery_->recover(hashed_candidate, session_index, backing_group, + core, std::move(on_recover_complete)); } @@ -2829,6 +2832,7 @@ namespace kagome::parachain { SessionIndex session, const HashedCandidateReceipt &hashed_candidate, GroupIndex backing_group, + std::optional core, bool distribute_assignment) { /// TODO(iceseer): don't launch approval work if the node is syncing. const auto &block_hash = indirect_cert.block_hash; @@ -2862,6 +2866,7 @@ namespace kagome::parachain { hashed_candidate, validator_index, block_hash, + core, backing_group); } else if (*approval_state == ApprovalOutcome::Approved) { issue_approval(hashed_candidate.getHash(), validator_index, block_hash); @@ -3207,6 +3212,15 @@ namespace kagome::parachain { candidate_receipt.descriptor.para_id, block_hash); + auto candidate_core_index = [&]() -> std::optional { + for (const auto &[core_index, h] : block_entry.candidates) { + if (candidate_hash == h) { + return core_index; + } + } + return std::nullopt; + }(); + if (auto claimed_core_indices = get_assignment_core_indices( indirect_cert.cert.kind, candidate_hash, block_entry)) { if (auto claimed_candidate_indices = cores_to_candidate_indices( @@ -3228,6 +3242,7 @@ namespace kagome::parachain { block_entry.session, candidate_entry.candidate, backing_group, + candidate_core_index, distribute_assignment); } else { diff --git a/core/parachain/approval/approval_distribution.hpp b/core/parachain/approval/approval_distribution.hpp index 281e6548d0..56d14f5404 100644 --- a/core/parachain/approval/approval_distribution.hpp +++ b/core/parachain/approval/approval_distribution.hpp @@ -233,9 +233,8 @@ namespace kagome::parachain { CandidateEntry(const network::CandidateReceipt &receipt, SessionIndex session_index, size_t approvals_size) - : CandidateEntry(HashedCandidateReceipt{receipt}, - session_index, - approvals_size) {} + : CandidateEntry( + HashedCandidateReceipt{receipt}, session_index, approvals_size) {} std::optional> approval_entry( const network::RelayHash &relay_hash) { @@ -746,6 +745,7 @@ namespace kagome::parachain { const HashedCandidateReceipt &hashed_receipt, ValidatorIndex validator_index, Hash block_hash, + std::optional core, GroupIndex backing_group); void issue_approval(const CandidateHash &candidate_hash, @@ -760,6 +760,7 @@ namespace kagome::parachain { SessionIndex session, const HashedCandidateReceipt &hashed_candidate, GroupIndex backing_group, + std::optional core, bool distribute_assignment); void runNewBlocks(approval::BlockApprovalMeta &&approval_meta, diff --git a/core/parachain/approval/state.hpp b/core/parachain/approval/state.hpp index d94e2b31c6..b4f9030bb1 100644 --- a/core/parachain/approval/state.hpp +++ b/core/parachain/approval/state.hpp @@ -113,7 +113,7 @@ namespace kagome::parachain::approval { : l ? *l : r ? *r : std::optional{}; - }; + } inline auto max_or_some(const std::optional &l, const std::optional &r) { @@ -121,7 +121,7 @@ namespace kagome::parachain::approval { : l ? *l : r ? *r : std::optional{}; - }; + } // Determining the amount of tranches required for approval or which // assignments are pending involves moving through a series of states while diff --git a/core/parachain/approval/store.hpp b/core/parachain/approval/store.hpp index aee6305394..8be5aed160 100644 --- a/core/parachain/approval/store.hpp +++ b/core/parachain/approval/store.hpp @@ -8,6 +8,7 @@ #include "utils/non_copyable.hpp" +#include #include namespace kagome::parachain { @@ -68,7 +69,7 @@ namespace kagome::parachain { std::reference_wrapper set(const K &k, V &&v) { assert(store_.size() < kDebugHardLimit); - return store_.template insert_or_assign(k, std::move(v)).first->second; + return store_.insert_or_assign(k, std::move(v)).first->second; } private: diff --git a/core/parachain/availability/chunks.hpp b/core/parachain/availability/chunks.hpp index 448215b3b7..f250c11935 100644 --- a/core/parachain/availability/chunks.hpp +++ b/core/parachain/availability/chunks.hpp @@ -63,6 +63,19 @@ namespace kagome::parachain { EC_CPP_TRY(data, encoder.reconstruct(_chunks)); return scale::decode(data); } + + inline outcome::result fromSystematicChunks( + size_t validators, const std::vector &chunks) { + EC_CPP_TRY(encoder, ec_cpp::create(validators)); + std::vector _chunks; + _chunks.resize(chunks.size()); + for (auto &chunk : chunks) { + _chunks[chunk.index] = chunk.chunk; + } + + EC_CPP_TRY(data, encoder.reconstruct_from_systematic(_chunks)); + return scale::decode(data); + } } // namespace kagome::parachain #undef ERASURE_CODING_ERROR diff --git a/core/parachain/availability/fetch/fetch_impl.cpp b/core/parachain/availability/fetch/fetch_impl.cpp index fb11623cf6..9dfc9ee18a 100644 --- a/core/parachain/availability/fetch/fetch_impl.cpp +++ b/core/parachain/availability/fetch/fetch_impl.cpp @@ -67,87 +67,89 @@ namespace kagome::parachain { while (not active.validators.empty()) { auto peer = query_audi_->get(active.validators.back()); active.validators.pop_back(); - if (peer) { - const auto &peer_id = peer.value().id; - auto peer_state = [&]() { - auto res = pm_->getPeerState(peer_id); - if (!res) { - SL_TRACE(log(), "From unknown peer {}", peer_id); - res = pm_->createDefaultPeerState(peer_id); - } - return res; - }(); + if (not peer.has_value()) { + continue; + } + const auto &peer_id = peer.value().id; - auto req_chunk_version = peer_state->get().req_chunk_version.value_or( - network::ReqChunkVersion::V1_obsolete); + auto peer_state = [&]() { + auto res = pm_->getPeerState(peer_id); + if (!res) { + SL_TRACE(log(), "From unknown peer {}", peer_id); + res = pm_->createDefaultPeerState(peer_id); + } + return res; + }(); - switch (req_chunk_version) { - case network::ReqChunkVersion::V2: - SL_DEBUG(log(), - "Sent request of chunk {} of candidate {} to peer {}", - active.chunk_index, - candidate_hash, - peer_id); - router_->getFetchChunkProtocol()->doRequest( - *peer, - {candidate_hash, active.chunk_index}, - [=, chunk_index{active.chunk_index}, weak{weak_from_this()}]( - outcome::result r) { - if (auto self = weak.lock()) { - if (r.has_value()) { - SL_DEBUG(log(), - "Result of request chunk {} of candidate {} to " - "peer {}: success", - chunk_index, - candidate_hash, - peer_id); - } else { - SL_DEBUG(log(), - "Result of request chunk {} of candidate {} to " - "peer {}: {}", - chunk_index, - candidate_hash, - peer_id, - r.error()); - } + auto req_chunk_version = peer_state->get().req_chunk_version.value_or( + network::ReqChunkVersion::V1_obsolete); - self->fetch(candidate_hash, std::move(r)); + switch (req_chunk_version) { + case network::ReqChunkVersion::V2: + SL_DEBUG(log(), + "Sent request of chunk {} of candidate {} to peer {}", + active.chunk_index, + candidate_hash, + peer_id); + router_->getFetchChunkProtocol()->doRequest( + peer_id, + {candidate_hash, active.chunk_index}, + [=, chunk_index{active.chunk_index}, weak{weak_from_this()}]( + outcome::result r) { + if (auto self = weak.lock()) { + if (r.has_value()) { + SL_DEBUG(log(), + "Result of request chunk {} of candidate {} to " + "peer {}: success", + chunk_index, + candidate_hash, + peer_id); + } else { + SL_DEBUG(log(), + "Result of request chunk {} of candidate {} to " + "peer {}: {}", + chunk_index, + candidate_hash, + peer_id, + r.error()); } - }); - break; - case network::ReqChunkVersion::V1_obsolete: - router_->getFetchChunkProtocolObsolete()->doRequest( - *peer, - {candidate_hash, active.chunk_index}, - [=, weak{weak_from_this()}]( - outcome::result r) { - if (auto self = weak.lock()) { - if (r.has_value()) { - auto response = visit_in_place( - r.value(), - [](const network::Empty &empty) - -> network::FetchChunkResponse { return empty; }, - [&](const network::ChunkObsolete &chunk_obsolete) - -> network::FetchChunkResponse { - return network::Chunk{ - .data = std::move(chunk_obsolete.data), - .chunk_index = active.chunk_index, - .proof = std::move(chunk_obsolete.proof), - }; - }); - self->fetch(candidate_hash, std::move(response)); - } else { - self->fetch(candidate_hash, r.as_failure()); - } + + self->fetch(candidate_hash, std::move(r)); + } + }); + break; + case network::ReqChunkVersion::V1_obsolete: + router_->getFetchChunkProtocolObsolete()->doRequest( + peer_id, + {candidate_hash, active.chunk_index}, + [=, weak{weak_from_this()}]( + outcome::result r) { + if (auto self = weak.lock()) { + if (r.has_value()) { + auto response = visit_in_place( + r.value(), + [](network::Empty &empty) + -> network::FetchChunkResponse { return empty; }, + [&](network::ChunkObsolete &chunk_obsolete) + -> network::FetchChunkResponse { + return network::Chunk{ + .data = std::move(chunk_obsolete.data), + .chunk_index = active.chunk_index, + .proof = std::move(chunk_obsolete.proof), + }; + }); + self->fetch(candidate_hash, std::move(response)); + } else { + self->fetch(candidate_hash, r.as_failure()); } - }); - break; - default: - UNREACHABLE; - } - return; + } + }); + break; + default: + UNREACHABLE; } + return; } SL_WARN(log(), "candidate={} chunk={} not found", diff --git a/core/parachain/availability/recovery/recovery.hpp b/core/parachain/availability/recovery/recovery.hpp index c67e0b52d7..2d8efc1d93 100644 --- a/core/parachain/availability/recovery/recovery.hpp +++ b/core/parachain/availability/recovery/recovery.hpp @@ -28,6 +28,7 @@ namespace kagome::parachain { virtual void recover(const HashedCandidateReceipt &hashed_receipt, SessionIndex session_index, std::optional backing_group, + std::optional core, Cb cb) = 0; }; } // namespace kagome::parachain diff --git a/core/parachain/availability/recovery/recovery_impl.cpp b/core/parachain/availability/recovery/recovery_impl.cpp index cdc2fdca11..ca35757913 100644 --- a/core/parachain/availability/recovery/recovery_impl.cpp +++ b/core/parachain/availability/recovery/recovery_impl.cpp @@ -7,13 +7,18 @@ #include "parachain/availability/recovery/recovery_impl.hpp" #include "application/chain_spec.hpp" +#include "authority_discovery/query/query.hpp" +#include "blockchain/block_tree.hpp" #include "network/impl/protocols/protocol_fetch_available_data.hpp" #include "network/impl/protocols/protocol_fetch_chunk.hpp" #include "network/impl/protocols/protocol_fetch_chunk_obsolete.hpp" #include "network/peer_manager.hpp" +#include "network/router.hpp" #include "parachain/availability/availability_chunk_index.hpp" #include "parachain/availability/chunks.hpp" #include "parachain/availability/proof.hpp" +#include "parachain/availability/store/store.hpp" +#include "runtime/runtime_api/parachain_host.hpp" namespace { constexpr auto fullRecoveriesStartedMetricName = @@ -26,17 +31,16 @@ namespace { const std::array results = {"success", "failure", "invalid"}; -#define incFullRecoveriesFinished(strategy, result) \ - do { \ - BOOST_ASSERT_MSG( \ - std::find(strategy_types.begin(), strategy_types.end(), strategy) \ - != strategy_types.end(), \ - "Unknown strategy type"); \ - BOOST_ASSERT_MSG( \ - std::find(results.begin(), results.end(), result) != results.end(), \ - "Unknown result type"); \ - full_recoveries_finished_.at(strategy).at(result)->inc(); \ - } while (false) + // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define incFullRecoveriesFinished(strategy, result) \ + [&] { \ + BOOST_ASSERT_MSG( \ + std::ranges::find(strategy_types, strategy) != strategy_types.end(), \ + "Unknown strategy type"); \ + BOOST_ASSERT_MSG(std::ranges::find(results, result) != results.end(), \ + "Unknown result type"); \ + full_recoveries_finished_.at(strategy).at(result)->inc(); \ + }() } // namespace @@ -69,6 +73,7 @@ namespace kagome::parachain { fullRecoveriesFinishedMetricName, "Total number of recoveries that finished"); + BOOST_ASSERT(chain_spec != nullptr); for (auto &strategy : strategy_types) { auto &metrics_for_strategy = full_recoveries_finished_[strategy]; for (auto &result : results) { @@ -85,7 +90,7 @@ namespace kagome::parachain { } void RecoveryImpl::remove(const CandidateHash &candidate) { - std::unique_lock lock{mutex_}; + Lock lock{mutex_}; active_.erase(candidate); cached_.erase(candidate); } @@ -93,8 +98,9 @@ namespace kagome::parachain { void RecoveryImpl::recover(const HashedCandidateReceipt &hashed_receipt, SessionIndex session_index, std::optional backing_group, + std::optional core_index, Cb cb) { - std::unique_lock lock{mutex_}; + Lock lock{mutex_}; const auto &receipt = hashed_receipt.get(); const auto &candidate_hash = hashed_receipt.getHash(); if (auto it = cached_.find(candidate_hash); it != cached_.end()) { @@ -127,217 +133,657 @@ namespace kagome::parachain { cb(_min.error()); return; } + auto _node_features = + parachain_api_->node_features(block.hash, session_index); + if (_node_features.has_error()) { + lock.unlock(); + cb(_node_features.error()); + return; + } + + ValidatorIndex start_pos = 0; + if (core_index.has_value()) { + if (availability_chunk_mapping_is_enabled(_node_features.value())) { + start_pos = core_index.value() * _min.value(); + } + } + Active active; active.erasure_encoding_root = receipt.descriptor.erasure_encoding_root; active.chunks_total = session->validators.size(); active.chunks_required = _min.value(); active.cb.emplace_back(std::move(cb)); active.validators = session->discovery_keys; - if (backing_group) { - active.order = session->validator_groups.at(*backing_group); - std::shuffle(active.order.begin(), active.order.end(), random_); + active.val2chunk = [n_validators{session->validators.size()}, start_pos]( + ValidatorIndex validator_index) -> ChunkIndex { + return (start_pos + validator_index) % n_validators; + }; + + if (backing_group.has_value()) { + active.validators_of_group = session->validator_groups.at(*backing_group); } active_.emplace(candidate_hash, std::move(active)); + + lock.unlock(); + full_from_bakers_recovery_prepare(candidate_hash); + } + + void RecoveryImpl::full_from_bakers_recovery_prepare( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + + auto it = active_.find(candidate_hash); + if (it == active_.end()) { + return; + } + auto &active = it->second; + + // Fill request order by validators of group + active.order = std::move(active.validators_of_group); + std::shuffle(active.order.begin(), active.order.end(), random_); + + // Is it possible to full recover from bakers + auto is_possible_to_recovery_from_bakers = not active.order.empty(); + lock.unlock(); - back(candidate_hash); + + if (is_possible_to_recovery_from_bakers) { + full_from_bakers_recovery(candidate_hash); + } else { + systematic_chunks_recovery_prepare(candidate_hash); + } } - void RecoveryImpl::back(const CandidateHash &candidate_hash) { - std::unique_lock lock{mutex_}; + void RecoveryImpl::full_from_bakers_recovery( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + auto it = active_.find(candidate_hash); if (it == active_.end()) { return; } auto &active = it->second; + + // Send requests while (not active.order.empty()) { auto peer = query_audi_->get(active.validators[active.order.back()]); active.order.pop_back(); if (peer) { - router_->getFetchAvailableDataProtocol()->doRequest( - peer->id, - candidate_hash, - [=, weak{weak_from_this()}]( - outcome::result r) { - auto self = weak.lock(); - if (not self) { - return; - } - self->back(candidate_hash, std::move(r)); - }); + send_fetch_available_data_request( + peer->id, candidate_hash, &RecoveryImpl::full_from_bakers_recovery); return; } } + lock.unlock(); + + // No known peer anymore to do full recovery + systematic_chunks_recovery_prepare(candidate_hash); + } + + void RecoveryImpl::systematic_chunks_recovery_prepare( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + + auto it = active_.find(candidate_hash); + if (it == active_.end()) { + return; + } + auto &active = it->second; + + // Refill request order basing chunks active.chunks = av_store_->getChunks(candidate_hash); - for (size_t i = 0; i < active.chunks_total; ++i) { - if (std::find_if(active.chunks.begin(), - active.chunks.end(), - [&](network::ErasureChunk &c) { return c.index == i; }) + for (size_t validator_index = 0; validator_index < active.chunks_total; + ++validator_index) { + auto chunk_index = active.val2chunk(validator_index); + + // Filter non systematic chunks + if (chunk_index >= active.chunks_required) { + continue; + } + + // Filter existing + if (std::ranges::find_if( + active.chunks, + [&](network::ErasureChunk &c) { return c.index == chunk_index; }) != active.chunks.end()) { continue; } - active.order.emplace_back(i); + active.order.emplace_back(validator_index); } std::shuffle(active.order.begin(), active.order.end(), random_); + active.queried.clear(); + + size_t systematic_chunk_count = [&] { + std::set sci; + for (auto &chunk : active.chunks) { + if (chunk.index < active.chunks_required) { + sci.emplace(chunk.index); + } + } + return sci.size(); + }(); + + // Is it possible to collect all systematic chunks? + bool is_possible_to_collect_systematic_chunks = + systematic_chunk_count + active.chunks_active + active.order.size() + >= active.chunks_required; + lock.unlock(); - chunk(candidate_hash); + + if (is_possible_to_collect_systematic_chunks) { + systematic_chunks_recovery(candidate_hash); + } else { + regular_chunks_recovery_prepare(candidate_hash); + } } - void RecoveryImpl::back( - const CandidateHash &candidate_hash, - outcome::result _backed) { - std::unique_lock lock{mutex_}; + void RecoveryImpl::systematic_chunks_recovery( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + auto it = active_.find(candidate_hash); if (it == active_.end()) { return; } auto &active = it->second; - if (_backed) { - if (auto data = boost::get(&_backed.value())) { - if (check(active, *data)) { - return done(lock, it, std::move(*data)); + + if (active.systematic_chunk_failed) { + lock.unlock(); + return regular_chunks_recovery(candidate_hash); + } + + size_t systematic_chunk_count = [&] { + std::set sci; + for (auto &chunk : active.chunks) { + if (chunk.index < active.chunks_required) { + sci.emplace(chunk.index); + } + } + return sci.size(); + }(); + + // All systematic chunks are collected + if (systematic_chunk_count >= active.chunks_required) { + auto data_res = + fromSystematicChunks(active.validators.size(), active.chunks); + [[unlikely]] if (data_res.has_error()) { + active.systematic_chunk_failed = true; + SL_DEBUG(logger_, + "Systematic data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + data_res.error()); + incFullRecoveriesFinished("systematic_chunks", "invalid"); + } else { + auto &data = data_res.value(); + auto res = check(active, data); + [[unlikely]] if (res.has_error()) { + active.systematic_chunk_failed = true; + SL_DEBUG(logger_, + "Systematic data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + res.error()); + incFullRecoveriesFinished("systematic_chunks", "invalid"); + } else { + SL_TRACE(logger_, + "Data recovery from systematic chunks complete. " + "(candidate={}, erasure_root={})", + candidate_hash, + active.erasure_encoding_root); + incFullRecoveriesFinished("systematic_chunks", "success"); + return done(lock, it, data); } } + lock.unlock(); + return regular_chunks_recovery_prepare(candidate_hash); + } + + // Is it possible to collect all systematic chunks? + bool is_possible_to_collect_systematic_chunks = + systematic_chunk_count + active.chunks_active + active.order.size() + >= active.chunks_required; + + if (not is_possible_to_collect_systematic_chunks) { + active.systematic_chunk_failed = true; + SL_TRACE( + logger_, + "Data recovery from systematic chunks is not possible. " + "(candidate={} collected={} requested={} in-queue={} required={})", + candidate_hash, + systematic_chunk_count, + active.chunks_active, + active.order.size(), + active.chunks_required); + incFullRecoveriesFinished("systematic_chunks", "failure"); + lock.unlock(); + return regular_chunks_recovery_prepare(candidate_hash); + } + + // Send requests + auto max = std::min(kParallelRequests, + active.chunks_required - systematic_chunk_count); + while (not active.order.empty() and active.chunks_active < max) { + auto validator_index = active.order.back(); + active.order.pop_back(); + auto peer = query_audi_->get(active.validators[validator_index]); + if (peer) { + ++active.chunks_active; + active.queried.emplace(validator_index); + send_fetch_chunk_request( + peer->id, + candidate_hash, + active.val2chunk(validator_index), // chunk_index + &RecoveryImpl::systematic_chunks_recovery); + } + } + + // No active request anymore for systematic chunks recovery + if (active.chunks_active == 0) { + active.systematic_chunk_failed = true; + SL_TRACE( + logger_, + "Data recovery from systematic chunks is not possible. " + "(candidate={} collected={} requested={} in-queue={} required={})", + candidate_hash, + systematic_chunk_count, + active.chunks_active, + active.order.size(), + active.chunks_required); + incFullRecoveriesFinished("systematic_chunks", "failure"); + lock.unlock(); + return regular_chunks_recovery_prepare(candidate_hash); } - lock.unlock(); - back(candidate_hash); } - void RecoveryImpl::chunk(const CandidateHash &candidate_hash) { - std::unique_lock lock{mutex_}; + void RecoveryImpl::regular_chunks_recovery_prepare( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + auto it = active_.find(candidate_hash); if (it == active_.end()) { return; } auto &active = it->second; + + // Update by existing chunks + auto chunks = av_store_->getChunks(candidate_hash); + for (auto &chunk : chunks) { + if (std::ranges::find_if( + active.chunks, + [&](const auto &c) { return c.index == chunk.index; }) + == active.chunks.end()) { + active.chunks.emplace_back(std::move(chunk)); + } + } + + // If existing chunks are already enough for regular chunk recovery if (active.chunks.size() >= active.chunks_required) { - auto _data = fromChunks(active.chunks_total, active.chunks); - if (_data) { - if (auto r = check(active, _data.value()); not r) { - _data = r.error(); + auto data_res = fromChunks(active.chunks_total, active.chunks); + [[unlikely]] if (data_res.has_error()) { + active.systematic_chunk_failed = true; + SL_DEBUG(logger_, + "Data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + data_res.error()); + incFullRecoveriesFinished("regular_chunks", "invalid"); + } else { + auto &data = data_res.value(); + auto res = check(active, data); + [[unlikely]] if (res.has_error()) { + active.systematic_chunk_failed = true; + SL_DEBUG(logger_, + "Data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + res.error()); + incFullRecoveriesFinished("regular_chunks", "invalid"); + data_res = res.as_failure(); + } else { + SL_TRACE(logger_, + "Data recovery from chunks complete. " + "(candidate={}, erasure_root={})", + candidate_hash, + active.erasure_encoding_root); + incFullRecoveriesFinished("regular_chunks", "success"); } } - return done(lock, it, _data); + return done(lock, it, data_res); } - if (active.chunks.size() + active.chunks_active + active.order.size() - < active.chunks_required) { + + // Refill request order by remaining validators + for (size_t validator_index = 0; validator_index < active.chunks_total; + ++validator_index) { + // Filter queried + if (active.queried.contains(validator_index)) { + continue; + } + + // Filter existing (only if mapping is not 1-to-1) + if (active.val2chunk(0) != 0) { + if (std::ranges::find_if( + active.chunks, + [chunk_index{active.val2chunk(validator_index)}]( + network::ErasureChunk &c) { + return c.index == chunk_index; + }) + != active.chunks.end()) { + continue; + } + } + + active.order.emplace_back(validator_index); + } + std::shuffle(active.order.begin(), active.order.end(), random_); + + // Is it possible to collect enough chunks for recovery? + auto is_possible_to_collect_required_chunks = + active.chunks.size() + active.chunks_active + active.order.size() + >= active.chunks_required; + + if (is_possible_to_collect_required_chunks) { + lock.unlock(); + return regular_chunks_recovery(candidate_hash); + } + + SL_TRACE(logger_, + "Data recovery from chunks is not possible. " + "(candidate={} collected={} requested={} in-queue={} required={})", + candidate_hash, + active.chunks.size(), + active.chunks_active, + active.order.size(), + active.chunks_required); + incFullRecoveriesFinished("regular_chunks", "failure"); + return done(lock, it, std::nullopt); + } + + void RecoveryImpl::regular_chunks_recovery( + const CandidateHash &candidate_hash) { + Lock lock{mutex_}; + + auto it = active_.find(candidate_hash); + if (it == active_.end()) { + return; + } + auto &active = it->second; + + // If existing chunks are already enough for regular chunk recovery + if (active.chunks.size() >= active.chunks_required) { + auto data_res = fromChunks(active.chunks_total, active.chunks); + [[unlikely]] if (data_res.has_error()) { + SL_DEBUG(logger_, + "Data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + data_res.error()); + incFullRecoveriesFinished("regular_chunks", "invalid"); + } else { + auto &data = data_res.value(); + auto res = check(active, data); + [[unlikely]] if (res.has_error()) { + SL_DEBUG(logger_, + "Data recovery error " + "(candidate={}, erasure_root={}): {}", + candidate_hash, + active.erasure_encoding_root, + res.error()); + incFullRecoveriesFinished("regular_chunks", "invalid"); + data_res = res.as_failure(); + } else { + SL_TRACE(logger_, + "Data recovery from chunks complete. " + "(candidate={}, erasure_root={})", + candidate_hash, + active.erasure_encoding_root); + incFullRecoveriesFinished("regular_chunks", "success"); + } + } + return done(lock, it, data_res); + } + + // Is it possible to collect enough chunks for recovery? + auto is_possible_to_collect_required_chunks = + active.chunks.size() + active.chunks_active + active.order.size() + >= active.chunks_required; + + if (not is_possible_to_collect_required_chunks) { + SL_TRACE( + logger_, + "Data recovery from chunks is not possible. " + "(candidate={} collected={} requested={} in-queue={} required={})", + candidate_hash, + active.chunks.size(), + active.chunks_active, + active.order.size(), + active.chunks_required); + incFullRecoveriesFinished("regular_chunks", "failure"); return done(lock, it, std::nullopt); } + + // Send requests auto max = std::min(kParallelRequests, active.chunks_required - active.chunks.size()); while (not active.order.empty() and active.chunks_active < max) { - auto index = active.order.back(); + auto validator_index = active.order.back(); active.order.pop_back(); - auto peer = query_audi_->get(active.validators[index]); - if (peer) { + auto peer = query_audi_->get(active.validators[validator_index]); + if (peer.has_value()) { ++active.chunks_active; + active.queried.emplace(validator_index); + send_fetch_chunk_request(peer->id, + candidate_hash, + active.val2chunk(validator_index), + &RecoveryImpl::regular_chunks_recovery); + } + } + + // No active request anymore for regular chunks recovery + if (active.chunks_active == 0) { + SL_TRACE( + logger_, + "Data recovery from chunks is not possible. " + "(candidate={} collected={} requested={} in-queue={} required={})", + candidate_hash, + active.chunks.size(), + active.chunks_active, + active.order.size(), + active.chunks_required); + incFullRecoveriesFinished("regular_chunks", "failure"); + return done(lock, it, std::nullopt); + } + } - const auto &peer_id = peer.value().id; - auto peer_state = [&]() { - auto res = pm_->getPeerState(peer_id); - if (!res) { - SL_TRACE(logger_, "From unknown peer {}", peer_id); - res = pm_->createDefaultPeerState(peer_id); + // Fetch available data protocol communication + void RecoveryImpl::send_fetch_available_data_request( + const libp2p::PeerId &peer_id, + const CandidateHash &candidate_hash, + SelfCb next_iteration) { + router_->getFetchAvailableDataProtocol()->doRequest( + peer_id, + candidate_hash, + [weak{weak_from_this()}, candidate_hash, peer_id, next_iteration]( + outcome::result response_res) { + if (auto self = weak.lock()) { + if (response_res.has_error()) { + SL_TRACE(self->logger_, + "Fetching available data for candidate {} from {} " + "returned error: {}", + candidate_hash, + peer_id, + response_res.error()); + } else if (boost::get(&response_res.value())) { + SL_TRACE(self->logger_, + "Fetching available data for candidate {} from {} " + "returned empty", + candidate_hash, + peer_id); + } + self->handle_fetch_available_data_response( + candidate_hash, std::move(response_res), next_iteration); } - return res; - }(); - - auto req_chunk_version = peer_state->get().req_chunk_version.value_or( - network::ReqChunkVersion::V1_obsolete); - - switch (req_chunk_version) { - case network::ReqChunkVersion::V2: - SL_DEBUG(logger_, - "Sent request of chunk {} of candidate {} to peer {}", - index, - candidate_hash, - peer_id); - router_->getFetchChunkProtocol()->doRequest( - peer->id, - {candidate_hash, index}, - [=, this, chunk_index{index}, weak{weak_from_this()}]( - outcome::result r) { - if (auto self = weak.lock()) { - if (r.has_value()) { - SL_DEBUG(logger_, - "Result of request chunk {} of candidate {} to " - "peer {}: success", - chunk_index, - candidate_hash, - peer_id); - } else { - SL_DEBUG(logger_, - "Result of request chunk {} of candidate {} to " - "peer {}: {}", - chunk_index, - candidate_hash, - peer_id, - r.error()); - } - - self->chunk(candidate_hash, index, std::move(r)); - } - }); - break; - case network::ReqChunkVersion::V1_obsolete: - router_->getFetchChunkProtocolObsolete()->doRequest( - peer->id, - {candidate_hash, index}, - [=, weak{weak_from_this()}]( - outcome::result r) { - if (auto self = weak.lock()) { - if (r.has_value()) { - auto response = visit_in_place( - r.value(), - [](const network::Empty &empty) - -> network::FetchChunkResponse { return empty; }, - [&](const network::ChunkObsolete &chunk_obsolete) - -> network::FetchChunkResponse { - return network::Chunk{ - .data = std::move(chunk_obsolete.data), - .chunk_index = index, - .proof = std::move(chunk_obsolete.proof), - }; - }); - self->chunk(candidate_hash, index, std::move(response)); - } else { - self->chunk(candidate_hash, index, r.as_failure()); - } - } - }); - break; - default: - UNREACHABLE; + }); + } + + void RecoveryImpl::handle_fetch_available_data_response( + const CandidateHash &candidate_hash, + outcome::result response_res, + SelfCb next_iteration) { + Lock lock{mutex_}; + + auto it = active_.find(candidate_hash); + if (it == active_.end()) { + return; + } + + auto &active = it->second; + + if (response_res.has_value()) { + if (auto data = boost::get(&response_res.value())) { + auto res = check(active, *data); + [[unlikely]] if (res.has_error()) { + incFullRecoveriesFinished("full_from_backers", "invalid"); + } else { + incFullRecoveriesFinished("full_from_backers", "success"); + return done(lock, it, std::move(*data)); } - return; } } - if (active.chunks_active == 0) { - done(lock, it, std::nullopt); + + lock.unlock(); + + (this->*next_iteration)(candidate_hash); + } + + void RecoveryImpl::send_fetch_chunk_request( + const libp2p::PeerId &peer_id, + const CandidateHash &candidate_hash, + ChunkIndex chunk_index, + SelfCb next_iteration) { + auto peer_state = [&]() { + auto res = pm_->getPeerState(peer_id); + if (!res) { + SL_TRACE(logger_, "From unknown peer {}", peer_id); + res = pm_->createDefaultPeerState(peer_id); + } + return res; + }(); + + auto req_chunk_version = peer_state->get().req_chunk_version.value_or( + network::ReqChunkVersion::V1_obsolete); + + switch (req_chunk_version) { + case network::ReqChunkVersion::V2: { + SL_DEBUG(logger_, + "Sent request of chunk {} of candidate {} to peer {}", + chunk_index, + candidate_hash, + peer_id); + router_->getFetchChunkProtocol()->doRequest( + peer_id, + {candidate_hash, chunk_index}, + [weak{weak_from_this()}, + candidate_hash, + chunk_index, + peer_id, + next_iteration]( + outcome::result response_res) { + if (auto self = weak.lock()) { + if (response_res.has_value()) { + SL_DEBUG(self->logger_, + "Result of request chunk {} of candidate {} to " + "peer {}: success", + chunk_index, + candidate_hash, + peer_id); + } else { + SL_DEBUG(self->logger_, + "Result of request chunk {} of candidate {} to " + "peer {}: {}", + chunk_index, + candidate_hash, + peer_id, + response_res.error()); + } + + self->handle_fetch_chunk_response( + candidate_hash, std::move(response_res), next_iteration); + } + }); + } break; + case network::ReqChunkVersion::V1_obsolete: { + router_->getFetchChunkProtocolObsolete()->doRequest( + peer_id, + {candidate_hash, chunk_index}, + [=, weak{weak_from_this()}]( + outcome::result + response_res) { + if (auto self = weak.lock()) { + if (response_res.has_value()) { + auto response = visit_in_place( + response_res.value(), + [](network::Empty &empty) -> network::FetchChunkResponse { + return empty; + }, + [&](network::ChunkObsolete &chunk_obsolete) + -> network::FetchChunkResponse { + return network::Chunk{ + .data = std::move(chunk_obsolete.data), + .chunk_index = chunk_index, + .proof = std::move(chunk_obsolete.proof), + }; + }); + self->handle_fetch_chunk_response( + candidate_hash, std::move(response), next_iteration); + } else { + self->handle_fetch_chunk_response(candidate_hash, + response_res.as_failure(), + next_iteration); + } + } + }); + } break; + default: + UNREACHABLE; } } - void RecoveryImpl::chunk( + void RecoveryImpl::handle_fetch_chunk_response( const CandidateHash &candidate_hash, - ChunkIndex index, - outcome::result _chunk) { - std::unique_lock lock{mutex_}; + outcome::result response_res, + SelfCb next_iteration) { + Lock lock{mutex_}; + auto it = active_.find(candidate_hash); if (it == active_.end()) { return; } auto &active = it->second; + --active.chunks_active; - if (_chunk) { - if (auto chunk2 = boost::get(&_chunk.value())) { - network::ErasureChunk chunk{ - std::move(chunk2->data), index, std::move(chunk2->proof)}; - if (checkTrieProof(chunk, active.erasure_encoding_root)) { - active.chunks.emplace_back(std::move(chunk)); + + if (response_res.has_value()) { + if (auto chunk = boost::get(&response_res.value())) { + network::ErasureChunk erasure_chunk{ + .chunk = std::move(chunk->data), + .index = chunk->chunk_index, + .proof = std::move(chunk->proof), + }; + if (checkTrieProof(erasure_chunk, active.erasure_encoding_root)) { + active.chunks.emplace_back(std::move(erasure_chunk)); } } } + lock.unlock(); - chunk(candidate_hash); + + (this->*next_iteration)(candidate_hash); } outcome::result RecoveryImpl::check(const Active &active, @@ -356,13 +802,9 @@ namespace kagome::parachain { const std::optional> &result_op) { if (result_op.has_value()) { auto &result = result_op.value(); - cached_.emplace(it->first, result); - - if (result.has_value()) { - // incFullRecoveriesFinished("unknown", "success"); // TODO fix strategy - } } + auto node = active_.extract(it); lock.unlock(); for (auto &cb : node.mapped().cb) { diff --git a/core/parachain/availability/recovery/recovery_impl.hpp b/core/parachain/availability/recovery/recovery_impl.hpp index c8f209eb07..dfa1929fca 100644 --- a/core/parachain/availability/recovery/recovery_impl.hpp +++ b/core/parachain/availability/recovery/recovery_impl.hpp @@ -10,22 +10,39 @@ #include #include +#include #include -#include "authority_discovery/query/query.hpp" -#include "blockchain/block_tree.hpp" #include "log/logger.hpp" #include "metrics/metrics.hpp" -#include "network/router.hpp" -#include "parachain/availability/store/store.hpp" -#include "runtime/runtime_api/parachain_host.hpp" namespace kagome::application { class ChainSpec; } +namespace kagome::authority_discovery { + class Query; +} + +namespace kagome::blockchain { + class BlockTree; +} + +namespace kagome::crypto { + class Hasher; +} + namespace kagome::network { class PeerManager; + class Router; +} // namespace kagome::network + +namespace kagome::parachain { + class AvailabilityStore; +} + +namespace kagome::runtime { + class ParachainHost; } namespace kagome::parachain { @@ -44,31 +61,63 @@ namespace kagome::parachain { void recover(const HashedCandidateReceipt &hashed_receipt, SessionIndex session_index, std::optional backing_group, + std::optional core, Cb cb) override; void remove(const CandidateHash &candidate) override; private: + using SelfCb = void (RecoveryImpl::*)(const CandidateHash &); + struct Active { storage::trie::RootHash erasure_encoding_root; - size_t chunks_total = 0; - size_t chunks_required = 0; + ChunkIndex chunks_total = 0; + ChunkIndex chunks_required = 0; std::vector cb; std::vector validators; + std::vector validators_of_group; std::vector order; + std::set queried; + bool systematic_chunk_failed = false; std::vector chunks; + std::function val2chunk; size_t chunks_active = 0; }; using ActiveMap = std::unordered_map; using Lock = std::unique_lock; - void back(const CandidateHash &candidate_hash); - void back(const CandidateHash &candidate_hash, - outcome::result _backed); - void chunk(const CandidateHash &candidate_hash); - void chunk(const CandidateHash &candidate_hash, - ChunkIndex index, - outcome::result _chunk); + // Full from bakers recovery strategy + void full_from_bakers_recovery_prepare(const CandidateHash &candidate_hash); + void full_from_bakers_recovery(const CandidateHash &candidate_hash); + + // Systematic recovery strategy + void systematic_chunks_recovery_prepare( + const CandidateHash &candidate_hash); + void systematic_chunks_recovery(const CandidateHash &candidate_hash); + + // Chunk recovery strategy + void regular_chunks_recovery_prepare(const CandidateHash &candidate_hash); + void regular_chunks_recovery(const CandidateHash &candidate_hash); + + // Fetch available data protocol communication + void send_fetch_available_data_request(const libp2p::PeerId &response_res, + const CandidateHash &candidate_hash, + SelfCb next_iteration); + void handle_fetch_available_data_response( + const CandidateHash &candidate_hash, + outcome::result response_res, + SelfCb next_iteration); + + // Fetch chunk protocol communication + void send_fetch_chunk_request(const libp2p::PeerId &peer_id, + const CandidateHash &candidate_hash, + ChunkIndex chunk_index, + SelfCb next_iteration); + void handle_fetch_chunk_response( + const CandidateHash &candidate_hash, + outcome::result response_res, + SelfCb next_iteration); + outcome::result check(const Active &active, const AvailableData &data); void done(Lock &lock, diff --git a/core/parachain/pvf/pool.cpp b/core/parachain/pvf/pool.cpp index 3c4255523d..d5780773f0 100644 --- a/core/parachain/pvf/pool.cpp +++ b/core/parachain/pvf/pool.cpp @@ -46,10 +46,10 @@ namespace kagome::parachain { std::shared_ptr module_factory, std::shared_ptr instrument) : pool_{std::make_shared( - app_config, - std::move(module_factory), - std::move(instrument), - app_config.parachainRuntimeInstanceCacheSize())} {} + app_config, + std::move(module_factory), + std::move(instrument), + app_config.parachainRuntimeInstanceCacheSize())} {} outcome::result PvfPool::precompile( const Hash256 &code_hash, diff --git a/core/parachain/pvf/secure_mode_precheck.hpp b/core/parachain/pvf/secure_mode_precheck.hpp index 615657cc48..9d4f5d0dc3 100644 --- a/core/parachain/pvf/secure_mode_precheck.hpp +++ b/core/parachain/pvf/secure_mode_precheck.hpp @@ -34,7 +34,7 @@ namespace kagome::parachain { static SecureModeSupport none() { return {false, false, false}; - }; + } bool isTotallySupported() const { return chroot && landlock && seccomp; diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index cbec906d41..92cccfa14c 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -3049,21 +3049,19 @@ namespace kagome::parachain { } template - void ParachainProcessorImpl::requestPoV( - const libp2p::peer::PeerInfo &peer_info, - const CandidateHash &candidate_hash, - F &&callback) { + void ParachainProcessorImpl::requestPoV(const libp2p::peer::PeerId &peer_id, + const CandidateHash &candidate_hash, + F &&callback) { /// TODO(iceseer): request PoV from validator, who seconded candidate /// But now we can assume, that if we received either `seconded` or `valid` /// from some peer, than we expect this peer has valid PoV, which we can /// request. - logger_->info("Requesting PoV.(candidate hash={}, peer={})", - candidate_hash, - peer_info.id); + logger_->info( + "Requesting PoV.(candidate hash={}, peer={})", candidate_hash, peer_id); auto protocol = router_->getReqPovProtocol(); - protocol->request(peer_info, candidate_hash, std::forward(callback)); + protocol->request(peer_id, candidate_hash, std::forward(callback)); } void ParachainProcessorImpl::kickOffValidationWork( @@ -3091,7 +3089,7 @@ namespace kagome::parachain { if (auto peer = query_audi_->get(authority_id)) { auto pvd{persisted_validation_data}; requestPoV( - *peer, + peer->id, candidate_hash, [candidate{attesting_data.candidate}, pvd{std::move(pvd)}, @@ -4407,8 +4405,7 @@ namespace kagome::parachain { } if (!parachain_state->get().our_index) { - logger_->template warn( - "We are not validators or we have no validator index."); + logger_->warn("We are not validators or we have no validator index."); return std::nullopt; } @@ -4459,7 +4456,7 @@ namespace kagome::parachain { if (stream_engine->reserveOutgoing(peer_id, protocol)) { protocol->newOutgoingStream( - libp2p::peer::PeerInfo{.id = peer_id, .addresses = {}}, + peer_id, [callback{std::forward(callback)}, protocol, peer_id, @@ -4531,7 +4528,7 @@ namespace kagome::parachain { logger_->info("Send my view.(peer={}, protocol={})", peer_id, protocol->protocolName()); - pm_->getStreamEngine()->template send( + pm_->getStreamEngine()->send( peer_id, protocol, std::make_shared< diff --git a/core/parachain/validator/parachain_processor.hpp b/core/parachain/validator/parachain_processor.hpp index 41267238b3..7f8ea6e3ae 100644 --- a/core/parachain/validator/parachain_processor.hpp +++ b/core/parachain/validator/parachain_processor.hpp @@ -755,7 +755,7 @@ namespace kagome::parachain { ParachainId para_id); template - void requestPoV(const libp2p::peer::PeerInfo &peer_info, + void requestPoV(const libp2p::peer::PeerId &peer_id, const CandidateHash &candidate_hash, F &&callback); diff --git a/core/primitives/code_substitutes.hpp b/core/primitives/code_substitutes.hpp index 430228976f..ab416d7a7b 100644 --- a/core/primitives/code_substitutes.hpp +++ b/core/primitives/code_substitutes.hpp @@ -32,7 +32,7 @@ namespace kagome::primitives { bool contains(const primitives::BlockInfo &block_info) const { return count(block_info.number) != 0 || count(block_info.hash) != 0; - }; + } bool contains(const primitives::BlockId &block_id) const { return count(block_id) != 0; diff --git a/core/primitives/strobe.hpp b/core/primitives/strobe.hpp index d504bc9922..19e20139ca 100644 --- a/core/primitives/strobe.hpp +++ b/core/primitives/strobe.hpp @@ -151,7 +151,7 @@ namespace kagome::primitives { std::copy(std::begin(other.raw_data), std::end(other.raw_data), std::begin(raw_data)); - }; + } Strobe &operator=(const Strobe &other) { std::copy(std::begin(other.raw_data), diff --git a/core/primitives/transaction_validity.cpp b/core/primitives/transaction_validity.cpp index 6967d33164..3826a55789 100644 --- a/core/primitives/transaction_validity.cpp +++ b/core/primitives/transaction_validity.cpp @@ -21,7 +21,9 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::primitives, InvalidTransaction::Kind, e) { return "General error to do with the transaction being outdated (e.g. " "nonce too low)"; case E::BadProof: - return "InvalidTransaction::BadProof. General error to do with the transaction's proofs (e.g. signature). May happen if finality lags behind best block (breaks transaction mortality encoding)."; + return "InvalidTransaction::BadProof. General error to do with the " + "transaction's proofs (e.g. signature). May happen if finality " + "lags behind best block (breaks transaction mortality encoding)."; case E::AncientBirthBlock: return "The transaction birth block is ancient"; case E::ExhaustsResources: diff --git a/core/runtime/runtime_api/impl/account_nonce_api.cpp b/core/runtime/runtime_api/impl/account_nonce_api.cpp index 6ea29592f2..345618be4c 100644 --- a/core/runtime/runtime_api/impl/account_nonce_api.cpp +++ b/core/runtime/runtime_api/impl/account_nonce_api.cpp @@ -21,6 +21,6 @@ namespace kagome::runtime { OUTCOME_TRY(ctx, executor_->ctx().ephemeralAt(block)); return executor_->call( ctx, "AccountNonceApi_account_nonce", account_id); - }; + } } // namespace kagome::runtime diff --git a/core/runtime/wasm_edge/register_host_api.hpp b/core/runtime/wasm_edge/register_host_api.hpp index e891f652d8..3d9784121a 100644 --- a/core/runtime/wasm_edge/register_host_api.hpp +++ b/core/runtime/wasm_edge/register_host_api.hpp @@ -140,8 +140,8 @@ namespace kagome::runtime::wasm_edge { } } catch (std::runtime_error &e) { - auto log = log::createLogger("HostApi", "runtime"); - SL_ERROR(log, "Host API call failed with error: {}", e.what()); + auto logger = log::createLogger("HostApi", "runtime"); + SL_ERROR(logger, "Host API call failed with error: {}", e.what()); return WasmEdge_Result_Fail; } return WasmEdge_Result_Success; diff --git a/core/scale/std_variant.hpp b/core/scale/std_variant.hpp index 1d3a481df7..6a3222c287 100644 --- a/core/scale/std_variant.hpp +++ b/core/scale/std_variant.hpp @@ -33,11 +33,10 @@ namespace scale { Stream &operator>>(Stream &stream, std::variant &variant) { uint8_t index; stream >> index; - using Decoder = - void (*)(Stream & stream, std::variant & variant); + using Decoder = void (*)(Stream &stream, std::variant &variant); constexpr Decoder decoders[]{ make_decoder()...}; decoders[index](stream, variant); return stream; } -} // namespace scale \ No newline at end of file +} // namespace scale diff --git a/core/scale/tie.hpp b/core/scale/tie.hpp index e5a194657e..cdc50693ba 100644 --- a/core/scale/tie.hpp +++ b/core/scale/tie.hpp @@ -26,8 +26,10 @@ return !operator==(r); \ } -#define SCALE_TIE_ONLY(...) \ - auto as_tie() { return std::tie(__VA_ARGS__); } \ +#define SCALE_TIE_ONLY(...) \ + auto as_tie() { \ + return std::tie(__VA_ARGS__); \ + } \ SCALE_TIE_EQ #define SCALE_TIE(N) \ diff --git a/core/scale/tie_hash.hpp b/core/scale/tie_hash.hpp index 894afd27ab..0d785a8b66 100644 --- a/core/scale/tie_hash.hpp +++ b/core/scale/tie_hash.hpp @@ -16,8 +16,10 @@ namespace scale { } } // namespace scale -#define SCALE_TIE_HASH_BOOST(type) \ - friend auto hash_value(const type &v) { return ::scale::tieHash(v); } +#define SCALE_TIE_HASH_BOOST(type) \ + friend auto hash_value(const type &v) { \ + return ::scale::tieHash(v); \ + } #define SCALE_TIE_HASH_STD(type) \ template <> \ diff --git a/core/telemetry/endpoint.hpp b/core/telemetry/endpoint.hpp index 0a45b7713d..616383bfa4 100644 --- a/core/telemetry/endpoint.hpp +++ b/core/telemetry/endpoint.hpp @@ -6,6 +6,8 @@ #pragma once +#include + #include "common/uri.hpp" namespace kagome::telemetry { @@ -21,11 +23,11 @@ namespace kagome::telemetry { const common::Uri &uri() const { return uri_; - }; + } uint8_t verbosity() { return verbosity_level_; - }; + } bool operator==(const TelemetryEndpoint &other) const { return uri_.to_string() == other.uri_.to_string() diff --git a/core/telemetry/impl/connection_impl.cpp b/core/telemetry/impl/connection_impl.cpp index 3b17fd5897..166504070a 100644 --- a/core/telemetry/impl/connection_impl.cpp +++ b/core/telemetry/impl/connection_impl.cpp @@ -150,8 +150,8 @@ namespace kagome::telemetry { } } - boost::beast::lowest_layer_type - &TelemetryConnectionImpl::stream_lowest_layer() { + boost::beast::lowest_layer_type & + TelemetryConnectionImpl::stream_lowest_layer() { return secure_ ? boost::beast::get_lowest_layer( *boost::relaxed_get(ws_)) : boost::beast::get_lowest_layer( diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index 9beb3caaf2..921b901cd7 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -151,7 +151,7 @@ Kagome DB Editor - a storage pruner. Allows to reduce occupied disk space. kagome-db-editor base-path/polkadot/db )"); std::cout << help; -}; +} outcome::result> persistent_batch( const std::unique_ptr &trie, const RootHash &hash) { @@ -196,7 +196,7 @@ void child_storage_root_hashes(const std::unique_ptr &batch, auto is_hash(const char *s) { return std::strlen(s) == common::Hash256::size() * 2 + 2 && std::equal(s, s + 2, "0x"); -}; +} int db_editor_main(int argc, const char **argv) { #if defined(BACKWARD_HAS_BACKTRACE) @@ -247,16 +247,16 @@ int db_editor_main(int argc, const char **argv) { std::make_shared(storage)); auto injector = di::make_injector( - di::bind.template to([](const auto &injector) { + di::bind.to([](const auto &injector) { return std::make_shared( injector.template create>(), injector.template create>(), injector.template create>()); }), - di::bind.template to(trie_node_tracker), - di::bind.template to( + di::bind.to(trie_node_tracker), + di::bind.to( std::shared_ptr(nullptr)), - di::bind.template to([](const auto &injector) { + di::bind.to([](const auto &injector) { return std::make_shared(kagome::crypto::blake2b<32>); }), di::bind.to(factory), diff --git a/core/utils/map.hpp b/core/utils/map.hpp index b77c3896c8..4fb899f749 100644 --- a/core/utils/map.hpp +++ b/core/utils/map.hpp @@ -11,6 +11,8 @@ #include #include +#include + namespace kagome::utils { template @@ -60,8 +62,8 @@ namespace kagome::utils { val = opt_ref->get(); } return val; - }; + } } // namespace kagome::utils -#endif // KAGOME_UTILS_MAP_HPP \ No newline at end of file +#endif // KAGOME_UTILS_MAP_HPP diff --git a/core/utils/struct_to_tuple.hpp b/core/utils/struct_to_tuple.hpp index 5ada6101fc..01d2126661 100644 --- a/core/utils/struct_to_tuple.hpp +++ b/core/utils/struct_to_tuple.hpp @@ -59,8 +59,10 @@ return std::make_tuple(REPEATY_REF(ONES, p)); \ } -#define TO_TUPLE1 \ - TO_TUPLE_N(1) else { return std::make_tuple(); } +#define TO_TUPLE1 \ + TO_TUPLE_N(1) else { \ + return std::make_tuple(); \ + } #define TO_TUPLE2 TO_TUPLE_N(2) else TO_TUPLE1 #define TO_TUPLE3 TO_TUPLE_N(3) else TO_TUPLE2 #define TO_TUPLE4 TO_TUPLE_N(4) else TO_TUPLE3 diff --git a/core/utils/weak_macro.hpp b/core/utils/weak_macro.hpp index 94831d6f9f..150981d7a1 100644 --- a/core/utils/weak_macro.hpp +++ b/core/utils/weak_macro.hpp @@ -6,8 +6,10 @@ #pragma once -#define WEAK_SELF \ - weak_self { weak_from_this() } +#define WEAK_SELF \ + weak_self { \ + weak_from_this() \ + } #define WEAK_LOCK(name) \ auto name = weak_##name.lock(); \ diff --git a/housekeeping/clang-tidy-diff.sh b/housekeeping/clang-tidy-diff.sh index 314d862af1..13658f1653 100755 --- a/housekeeping/clang-tidy-diff.sh +++ b/housekeeping/clang-tidy-diff.sh @@ -6,8 +6,14 @@ # BUILD_DIR="${BUILD_DIR:-build}" +CLANG_TIDY_BIN="${CLANG_TIDY_BIN:-clang-tidy}" +CI="${CI:-false}" + +if [ "${CI}" == "true" ]; then + git fetch origin master +fi cd $(dirname $0)/.. # exclude WAVM because on CI clang-tidy is run on a WasmEdge build -git diff -U0 origin/master -- . ':!core/runtime/wavm' | clang-tidy-diff.py -p1 -path $BUILD_DIR -iregex '(core|node)\/.*\.(h|c|hpp|cpp)' -clang-tidy-binary clang-tidy-16 | tee clang-tidy.log +git diff -U0 origin/master -- . ':!core/runtime/wavm' | clang-tidy-diff.py -p1 -path ${BUILD_DIR} -iregex '(core|node)\/.*\.(h|c|hpp|cpp)' -clang-tidy-binary ${CLANG_TIDY_BIN} | tee clang-tidy.log ! grep ': error:' clang-tidy.log diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index 4a560ff1c1..7f4856e96f 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -39,6 +39,7 @@ REGION ?= europe-north1 # CI Variables IS_MAIN_OR_TAG ?= false GIT_REF_NAME ?= +CI ?= false export DOCKER_BUILDKIT=1 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson @@ -119,6 +120,55 @@ kagome_dev_docker_build: fi; \ docker stop $$CONTAINER_NAME +kagome_dev_docker_build_tidy: + $(MAKE) get_versions + mkdir -p \ + $(CACHE_DIR)/.cargo/git \ + $(CACHE_DIR)/.cargo/registry \ + $(CACHE_DIR)/.hunter \ + $(CACHE_DIR)/.cache/ccache ; \ + CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ + SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ + DOCKER_EXEC_RESULT=0 ; \ + echo "Build type: $(BUILD_TYPE)"; \ + docker run -d --name $$CONTAINER_NAME \ + --platform $(PLATFORM) \ + --entrypoint "/bin/bash" \ + -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ + -e BUILD_TYPE=$(BUILD_TYPE) \ + -e BUILD_THREADS=$$BUILD_THREADS \ + -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ + -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ + -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ + -e CTEST_OUTPUT_ON_FAILURE=$(CTEST_OUTPUT_ON_FAILURE) \ + -v $$(pwd)/../../../../kagome:/opt/kagome \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ + -v $(CACHE_DIR)/.cargo/git:/root/.cargo/git \ + -v $(CACHE_DIR)/.cargo/registry:/root/.cargo/registry \ + -v $(CACHE_DIR)/.hunter:/root/.hunter \ + -v $(CACHE_DIR)/.cache/ccache:/root/.cache/ccache \ + $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(MINIDEB_TAG) \ + -c "tail -f /dev/null"; \ + docker exec -t $$CONTAINER_NAME /bin/bash -c \ + "clang --version && \ + cd /opt/kagome && \ + git config --global --add safe.directory /opt/kagome && \ + git config --global --add safe.directory /root/.hunter/_Base/Cache/meta && \ + source /venv/bin/activate && \ + git submodule update --init && \ + echo \"Building in $$(pwd)\" && \ + cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ + cmake --build \"$(BUILD_DIR)\" --target generated -- -j${BUILD_THREADS} && \ + cd /opt/kagome/ && export CI='$(CI)' && ./housekeeping/clang-tidy-diff.sh \ + " || DOCKER_EXEC_RESULT=$$? ; \ + if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ + echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ + docker stop $$CONTAINER_NAME; \ + exit $$DOCKER_EXEC_RESULT; \ + fi; \ + docker stop $$CONTAINER_NAME + upload_apt_package: SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ diff --git a/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile index 63be602aa3..a89308aeca 100644 --- a/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile +++ b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile @@ -4,6 +4,9 @@ ARG BASE_IMAGE ARG RUST_VERSION ARG ARCHITECTURE=x86_64 +ARG DEBIAN_VERSION=bookworm +ARG LLVM_VERSION=19 + FROM ${BASE_IMAGE} ARG AUTHOR @@ -13,11 +16,28 @@ LABEL org.opencontainers.image.description="Kagome builder image" SHELL ["/bin/bash", "-c"] +ARG DEBIAN_VERSION +ENV DEBIAN_VERSION=${DEBIAN_VERSION} +ARG LLVM_VERSION +ENV LLVM_VERSION=${LLVM_VERSION} + +RUN install_packages \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget + +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/${DEBIAN_VERSION}/ llvm-toolchain-${DEBIAN_VERSION}-${LLVM_VERSION} main" | \ + tee -a /etc/apt/sources.list.d/llvm.list + RUN install_packages \ build-essential \ ccache \ - clang-format-16 \ - clang-tidy-16 \ + clang-format-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + libclang-rt-${LLVM_VERSION}-dev \ + llvm-${LLVM_VERSION}-dev \ curl \ dpkg-dev \ g++-12 \ @@ -28,11 +48,9 @@ RUN install_packages \ gpg \ gpg-agent \ lcov \ - libclang-rt-16-dev \ libgmp10 \ libnsl-dev \ libseccomp-dev \ - llvm-16-dev \ make \ mold \ nano \ @@ -42,7 +60,6 @@ RUN install_packages \ software-properties-common \ unzip \ vim \ - wget \ zlib1g-dev ARG RUST_VERSION @@ -59,8 +76,8 @@ RUN /venv/bin/pip install --no-cache-dir cmake==3.25 scikit-build requests gitpy ENV HUNTER_PYTHON_LOCATION=/venv/bin/python3 -ENV LLVM_ROOT=/usr/lib/llvm-16 -ENV LLVM_DIR=/usr/lib/llvm-16/lib/cmake/llvm/ +ENV LLVM_ROOT=/usr/lib/llvm-${LLVM_VERSION} +ENV LLVM_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/ ENV PATH=${LLVM_ROOT}/bin:${LLVM_ROOT}/share/clang:${PATH} ENV CC=gcc-12 ENV CXX=g++-12 @@ -68,10 +85,10 @@ ENV CXX=g++-12 RUN update-alternatives --install /usr/bin/python python /venv/bin/python3 90 && \ update-alternatives --install /usr/bin/python python /usr/bin/python3 80 && \ \ - update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-16 50 && \ - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-16 50 && \ - update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-16/bin/clang-16 50 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 50 && \ + update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${LLVM_VERSION} 50 && \ + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_VERSION} 50 && \ + update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-${LLVM_VERSION}/bin/clang-${LLVM_VERSION} 50 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 50 && \ \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90 && \ diff --git a/test/core/application/app_state_manager_test.cpp b/test/core/application/app_state_manager_test.cpp index 7d4747661d..ad754c9381 100644 --- a/test/core/application/app_state_manager_test.cpp +++ b/test/core/application/app_state_manager_test.cpp @@ -187,17 +187,17 @@ struct UnderControlObject { bool prepare() { tag = 1; return p(); - }; + } bool start() { tag = 2; return l(); - }; + } void stop() { tag = 3; s(); - }; + } }; /** diff --git a/test/core/blockchain/block_tree_test.cpp b/test/core/blockchain/block_tree_test.cpp index bb98ff2d8e..b5371bd2bd 100644 --- a/test/core/blockchain/block_tree_test.cpp +++ b/test/core/blockchain/block_tree_test.cpp @@ -323,7 +323,7 @@ struct BlockTreeTest : public testing::Test { digest.emplace_back(Seal{{primitives::kBabeEngineId, encoded_seal}}); return digest; - }; + } const BlockInfo kGenesisBlockInfo{ 0ul, BlockHash::fromString("genesis_block___________________").value()}; diff --git a/test/core/consensus/babe/babe_test.cpp b/test/core/consensus/babe/babe_test.cpp index 20114a13c9..ba8b39baa2 100644 --- a/test/core/consensus/babe/babe_test.cpp +++ b/test/core/consensus/babe/babe_test.cpp @@ -149,7 +149,7 @@ static Digest make_digest(SlotNumber slot, AuthorityIndex authority_index = 0) { SealDigest{kagome::primitives::kBabeEngineId, encoded_seal}); return digest; -}; +} class BabeWrapper : public Babe { public: diff --git a/test/core/consensus/timeline/timeline_test.cpp b/test/core/consensus/timeline/timeline_test.cpp index fa2e1296b0..c7b0cbeb4c 100644 --- a/test/core/consensus/timeline/timeline_test.cpp +++ b/test/core/consensus/timeline/timeline_test.cpp @@ -107,7 +107,7 @@ static Digest make_digest(SlotNumber slot) { SealDigest{kagome::primitives::kBabeEngineId, encoded_seal}); return digest; -}; +} class TimelineTest : public testing::Test { public: diff --git a/test/core/parachain/CMakeLists.txt b/test/core/parachain/CMakeLists.txt index 6e7ec55887..006df192cd 100644 --- a/test/core/parachain/CMakeLists.txt +++ b/test/core/parachain/CMakeLists.txt @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # +add_subdirectory(availability) + addtest(parachain_test pvf_test.cpp assignments.cpp diff --git a/test/core/parachain/availability/CMakeLists.txt b/test/core/parachain/availability/CMakeLists.txt new file mode 100644 index 0000000000..39e420db8d --- /dev/null +++ b/test/core/parachain/availability/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +addtest(recovery_test + recovery_test.cpp +) + +target_link_libraries(recovery_test + validator_parachain + dummy_error +) diff --git a/test/core/parachain/availability/recovery_test.cpp b/test/core/parachain/availability/recovery_test.cpp new file mode 100644 index 0000000000..a3cfacf10f --- /dev/null +++ b/test/core/parachain/availability/recovery_test.cpp @@ -0,0 +1,423 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "parachain/availability/recovery/recovery_impl.hpp" + +#include + +#include "crypto/random_generator/boost_generator.hpp" +#include "mock/core/application/chain_spec_mock.hpp" +#include "mock/core/authority_discovery/query_mock.hpp" +#include "mock/core/blockchain/block_tree_mock.hpp" +#include "mock/core/crypto/hasher_mock.hpp" +#include "mock/core/network/peer_manager_mock.hpp" +#include "mock/core/network/router_mock.hpp" +#include "mock/core/parachain/availability_store_mock.hpp" +#include "mock/core/runtime/parachain_host_mock.hpp" +#include "parachain/availability/chunks.hpp" +#include "parachain/availability/proof.hpp" +#include "testutil/literals.hpp" +#include "testutil/outcome.hpp" +#include "testutil/prepare_loggers.hpp" + +using kagome::Buffer; +using kagome::application::ChainSpecMock; +using kagome::authority_discovery::QueryMock; +using kagome::blockchain::BlockTreeMock; +using kagome::common::Buffer; +using kagome::crypto::BoostRandomGenerator; +using kagome::crypto::HasherMock; +using kagome::network::CandidateHash; +using kagome::network::CandidateReceipt; +using kagome::network::Chunk; +using kagome::network::PeerManagerMock; +using kagome::network::PeerState; +using kagome::network::RouterMock; +using kagome::parachain::AvailabilityStoreMock; +using kagome::parachain::CoreIndex; +using kagome::parachain::GroupIndex; +using kagome::parachain::Recovery; +using kagome::parachain::RecoveryImpl; +using kagome::parachain::SessionIndex; +using kagome::parachain::ValidatorId; +using kagome::primitives::AuthorityDiscoveryId; +using kagome::primitives::BlockInfo; +using kagome::runtime::AvailableData; +using kagome::runtime::ParachainHost; +using kagome::runtime::ParachainHostMock; +using kagome::runtime::SessionInfo; + +using kagome::parachain::makeTrieProof; +using kagome::parachain::toChunks; + +using libp2p::PeerId; +using libp2p::peer::PeerInfo; + +using testing::_; +using testing::Args; +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; +using testing::Truly; +using testing::WithArgs; + +class RecoveryTest : public testing::Test { + public: + static void SetUpTestCase() { + testutil::prepareLoggers(soralog::Level::TRACE); + } + + void SetUp() override { + size_t data_size = 8; + original_data.resize(data_size); + random_generator.fillRandomly(original_data); + + original_chunks = toChunks(n_validators, original_available_data).value(); + receipt.descriptor.erasure_encoding_root = makeTrieProof(original_chunks); + + session = SessionInfo{}; + + chain_spec = std::make_shared(); + static std::string chain_type{"network"}; + EXPECT_CALL(*chain_spec, chainType()).WillRepeatedly(ReturnRef(chain_type)); + + hasher = std::make_shared(); + + block_tree = std::make_shared(); + ON_CALL(*block_tree, bestBlock()).WillByDefault(Return(best_block)); + + parachain_api = std::make_shared(); + ON_CALL(*parachain_api, session_info(best_block.hash, session_index)) + .WillByDefault(Invoke([&] { return session; })); + ON_CALL(*parachain_api, node_features(best_block.hash, session_index)) + .WillByDefault(Invoke([&] { + scale::BitVec nf{}; + nf.bits.resize(ParachainHost::NodeFeatureIndex::FirstUnassigned); + nf.bits[ParachainHost::NodeFeatureIndex::AvailabilityChunkMapping] = + true; + return std::optional{nf}; + })); + + av_store = std::make_shared(); + query_audi = std::make_shared(); + + router = std::make_shared(); + router->setReturningMockedProtocols(); + + auto fetch_available_data = router->getMockedFetchAvailableDataProtocol(); + ON_CALL(*fetch_available_data, doRequest(_, _, _)) + .WillByDefault(WithArgs<0, 1, 2>([&](auto &&...args) { + fetch_available_data_requests.emplace( + std::forward(args)...); + })); + + auto fetch_chunk = router->getMockedFetchChunkProtocol(); + ON_CALL(*fetch_chunk, doRequest(_, _, _)) + .WillByDefault(WithArgs<0, 1, 2>([&](auto &&...args) { + fetch_chunk_requests.emplace(std::forward(args)...); + })); + + auto fetch_chunk_obsolete = router->getMockedFetchChunkProtocolObsolete(); + ON_CALL(*fetch_chunk_obsolete, doRequest(_, _, _)) + .WillByDefault(WithArgs<0, 1, 2>([&](auto &&...args) { + fetch_chunk_obsolete_requests.emplace( + std::forward(args)...); + })); + + peer_manager = std::make_shared(); + + recovery = std::make_shared(chain_spec, + hasher, + block_tree, + parachain_api, + av_store, + query_audi, + router, + peer_manager); + + auto &val_group_0 = session.validator_groups.emplace_back(); + for (size_t i = 0; i < n_validators; ++i) { + auto s = fmt::format("Validator#{:<{}}", i, ValidatorId::size() - 10); + ValidatorId validator_id = + ValidatorId::fromSpan(Buffer::fromString(s)).value(); + + s = fmt::format("Authority#{:<{}}", i, AuthorityDiscoveryId::size() - 10); + AuthorityDiscoveryId audi_id = + AuthorityDiscoveryId::fromSpan(Buffer::fromString(s)).value(); + + s = fmt::format("Peer#{}", i); + PeerId peer_id = operator""_peerid(s.data(), s.size()); + + session.validators.push_back(validator_id); + session.discovery_keys.push_back(audi_id); + val_group_0.push_back(i); + + ON_CALL(*query_audi, get(audi_id)) + .WillByDefault(Return(PeerInfo{.id = peer_id, .addresses = {}})); + ON_CALL(*query_audi, get(peer_id)).WillByDefault(Return(audi_id)); + ON_CALL(*peer_manager, getPeerState(peer_id)).WillByDefault(Invoke([&] { + return std::ref(peer_state); + })); + } + + callback = std::make_shared>)>>(); + } + + BoostRandomGenerator random_generator; + + size_t n_validators = 6; + Buffer original_data; + AvailableData original_available_data{}; + std::vector original_chunks; + + CandidateReceipt receipt; + BlockInfo best_block{}; + SessionIndex session_index; + SessionInfo session{}; + PeerState peer_state; + + std::queue)>>> + fetch_available_data_requests; + + std::queue)>>> + fetch_chunk_requests; + + std::queue)>>> + fetch_chunk_obsolete_requests; + + std::shared_ptr chain_spec; + std::shared_ptr hasher; + std::shared_ptr block_tree; + std::shared_ptr parachain_api; + std::shared_ptr av_store; + std::shared_ptr query_audi; + std::shared_ptr router; + std::shared_ptr peer_manager; + + std::shared_ptr>)>> + callback; + + std::shared_ptr recovery; +}; + +TEST_F(RecoveryTest, FullFromBakers_NoGroup) { + std::optional backing_group = std::nullopt; + std::optional core = std::nullopt; // value doesn't matter + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + // No one fetch available data request was sent + ASSERT_TRUE(fetch_available_data_requests.empty()); +} + +TEST_F(RecoveryTest, FullFromBakers_Success) { + std::optional backing_group = 0; // any non nullopt value + std::optional core = std::nullopt; // value doesn't matter + + std::optional> available_data_res_opt; + + EXPECT_CALL(*callback, Call(_)) + .WillOnce(WithArgs<0>( + [&](std::optional> x) mutable { + available_data_res_opt = std::move(x); + })); + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + while (not fetch_available_data_requests.empty()) { + auto [peer_id, req, cb] = std::move(fetch_available_data_requests.front()); + fetch_available_data_requests.pop(); + cb(original_available_data); + } + + testing::Mock::VerifyAndClear(callback.get()); + + ASSERT_TRUE(available_data_res_opt.has_value()); + ASSERT_OUTCOME_SUCCESS(available_data, available_data_res_opt.value()); + ASSERT_EQ(available_data, original_available_data); +} + +TEST_F(RecoveryTest, SystematicChunks_NoCore) { + peer_state.req_chunk_version = kagome::network::ReqChunkVersion::V2; + std::optional backing_group{}; // nullopt to skip full recovery + std::optional core = std::nullopt; + + std::optional> available_data_res_opt; + + EXPECT_CALL(*callback, Call(_)) + .WillOnce(WithArgs<0>( + [&](std::optional> x) mutable { + available_data_res_opt = std::move(x); + })); + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + // Full recovery from bakers is skipped, 'cause backing_group is unknown + ASSERT_TRUE(fetch_available_data_requests.empty()); + + // Trying to do systematic chunks recovery + while (not fetch_chunk_requests.empty()) { + auto &[peer_id, req, cb] = fetch_chunk_requests.front(); + const auto &ec_chunk = original_chunks[req.chunk_index]; + Chunk chunk{ + .data = ec_chunk.chunk, + .chunk_index = ec_chunk.index, + .proof = ec_chunk.proof, + }; + cb(chunk); + fetch_chunk_requests.pop(); + } + + testing::Mock::VerifyAndClear(callback.get()); + + ASSERT_TRUE(available_data_res_opt.has_value()); + ASSERT_OUTCOME_SUCCESS(available_data, available_data_res_opt.value()); + ASSERT_EQ(available_data, original_available_data); +} + +TEST_F(RecoveryTest, SystematicChunks_Success) { + peer_state.req_chunk_version = kagome::network::ReqChunkVersion::V2; + std::optional backing_group{}; // nullopt to skip full recovery + std::optional core = 0; + + std::optional> available_data_res_opt; + + EXPECT_CALL(*callback, Call(_)) + .WillOnce(WithArgs<0>( + [&](std::optional> x) mutable { + available_data_res_opt = std::move(x); + })); + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + // Full recovery from bakers is skipped, 'cause backing_group is unknown + ASSERT_TRUE(fetch_available_data_requests.empty()); + + // Trying to do systematic chunks recovery + while (not fetch_chunk_requests.empty()) { + auto &[peer_id, req, cb] = fetch_chunk_requests.front(); + const auto &ec_chunk = original_chunks[req.chunk_index]; + Chunk chunk{ + .data = ec_chunk.chunk, + .chunk_index = ec_chunk.index, + .proof = ec_chunk.proof, + }; + cb(chunk); + fetch_chunk_requests.pop(); + } + + testing::Mock::VerifyAndClear(callback.get()); + + ASSERT_TRUE(available_data_res_opt.has_value()); + ASSERT_OUTCOME_SUCCESS(available_data, available_data_res_opt.value()); + ASSERT_EQ(available_data, original_available_data); +} + +TEST_F(RecoveryTest, RegularChunks_Success) { + peer_state.req_chunk_version = kagome::network::ReqChunkVersion::V2; + std::optional backing_group{}; // nullopt to skip full recovery + std::optional core = 0; + + std::optional> available_data_res_opt; + + EXPECT_CALL(*callback, Call(_)) + .WillOnce(WithArgs<0>( + [&](std::optional> x) mutable { + available_data_res_opt = std::move(x); + })); + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + // Full recovery from bakers is skipped, 'cause backing_group is unknown + ASSERT_TRUE(fetch_available_data_requests.empty()); + + // Trying to do systematic chunks recovery, but one chunk unavailable + while (not fetch_chunk_requests.empty()) { + auto &[peer_id, req, cb] = fetch_chunk_requests.front(); + const auto &ec_chunk = original_chunks[req.chunk_index]; + if (ec_chunk.index != 0) { + Chunk chunk{ + .data = ec_chunk.chunk, + .chunk_index = ec_chunk.index, + .proof = ec_chunk.proof, + }; + cb(chunk); + } else { + cb(kagome::network::Empty{}); + } + fetch_chunk_requests.pop(); + } + + testing::Mock::VerifyAndClear(callback.get()); + + ASSERT_TRUE(available_data_res_opt.has_value()); + ASSERT_OUTCOME_SUCCESS(available_data, available_data_res_opt.value()); + ASSERT_EQ(available_data, original_available_data); +} + +TEST_F(RecoveryTest, Failure) { + peer_state.req_chunk_version = kagome::network::ReqChunkVersion::V2; + std::optional backing_group = 0; + std::optional core = 0; + + std::optional> available_data_res_opt; + + EXPECT_CALL(*callback, Call(_)) + .WillOnce(WithArgs<0>( + [&](std::optional> x) mutable { + available_data_res_opt = std::move(x); + })); + + recovery->recover( + receipt, session_index, backing_group, core, callback->AsStdFunction()); + + // Trying to recover full from bakers, but all return no-data + while (not fetch_available_data_requests.empty()) { + auto &[peer_id, req, cb] = fetch_available_data_requests.front(); + cb(kagome::network::Empty{}); + fetch_available_data_requests.pop(); + } + + // Trying to do chunks recovery, but not enough number chunks + while (not fetch_chunk_requests.empty()) { + auto &[peer_id, req, cb] = fetch_chunk_requests.front(); + const auto &ec_chunk = original_chunks[req.chunk_index]; + if (ec_chunk.index == 0) { + Chunk chunk{ + .data = ec_chunk.chunk, + .chunk_index = ec_chunk.index, + .proof = ec_chunk.proof, + }; + cb(chunk); + } else { + cb(kagome::network::Empty{}); + } + fetch_chunk_requests.pop(); + } + + testing::Mock::VerifyAndClear(callback.get()); + + ASSERT_FALSE(available_data_res_opt.has_value()); +} diff --git a/test/deps/di_test.cpp b/test/deps/di_test.cpp index 653536d5db..3863696a64 100644 --- a/test/deps/di_test.cpp +++ b/test/deps/di_test.cpp @@ -24,7 +24,7 @@ class example { example(aggregate a, const ctor &c) { EXPECT_EQ(87.0, a.d); EXPECT_EQ(42, c.i); - }; + } virtual void func() {} }; diff --git a/test/mock/core/authority_discovery/query_mock.hpp b/test/mock/core/authority_discovery/query_mock.hpp new file mode 100644 index 0000000000..535d09da6e --- /dev/null +++ b/test/mock/core/authority_discovery/query_mock.hpp @@ -0,0 +1,28 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "authority_discovery/query/query.hpp" + +#include + +namespace kagome::authority_discovery { + + class QueryMock : public Query { + public: + MOCK_METHOD(std::optional, + get, + (const primitives::AuthorityDiscoveryId &), + (const, override)); + + MOCK_METHOD(std::optional, + get, + (const libp2p::peer::PeerId &), + (const, override)); + }; + +} // namespace kagome::authority_discovery diff --git a/test/mock/core/network/protocol_base_mock.hpp b/test/mock/core/network/protocol_base_mock.hpp index 210644bd9e..8b3d2e7917 100644 --- a/test/mock/core/network/protocol_base_mock.hpp +++ b/test/mock/core/network/protocol_base_mock.hpp @@ -23,16 +23,16 @@ namespace kagome::network { MOCK_METHOD( void, newOutgoingStream, - (const PeerInfo &, + (const PeerId &, const std::function>)> &)); void newOutgoingStream( - const PeerInfo &peer_info, + const PeerId &peer_id, std::function>)> &&handler) override { const auto h = std::move(handler); - newOutgoingStream(peer_info, h); + newOutgoingStream(peer_id, h); } }; diff --git a/test/mock/core/network/protocol_mocks.hpp b/test/mock/core/network/protocol_mocks.hpp new file mode 100644 index 0000000000..ad00c273bb --- /dev/null +++ b/test/mock/core/network/protocol_mocks.hpp @@ -0,0 +1,52 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "network/impl/protocols/protocol_fetch_available_data.hpp" +#include "network/impl/protocols/protocol_fetch_chunk.hpp" +#include "network/impl/protocols/protocol_fetch_chunk_obsolete.hpp" +#include "network/impl/protocols/request_response_protocol.hpp" + +#include "mock/core/network/protocol_base_mock.hpp" +#include "mock/core/network/protocol_mocks.hpp" + +namespace kagome::network { + + template + requires(requires { + typename ProtocolInterface::RequestType; + typename ProtocolInterface::ResponseType; + }) + class RequestResponseProtocolMock + : public ProtocolInterface, + public ProtocolBaseMock, + virtual public RequestResponseProtocol< + typename ProtocolInterface::RequestType, + typename ProtocolInterface::ResponseType> { + public: + MOCK_METHOD( + void, + doRequest, + (const PeerId &, + typename ProtocolInterface::RequestType, + std::function)> &&), + ()); + }; + + using FetchChunkProtocolMock = + RequestResponseProtocolMock; + + using FetchChunkProtocolObsoleteMock = + RequestResponseProtocolMock; + + using FetchAvailableDataProtocolMock = + RequestResponseProtocolMock; + +} // namespace kagome::network diff --git a/test/mock/core/network/router_mock.hpp b/test/mock/core/network/router_mock.hpp index 01fe6936bb..e3186f376d 100644 --- a/test/mock/core/network/router_mock.hpp +++ b/test/mock/core/network/router_mock.hpp @@ -10,6 +10,8 @@ #include +#include "mock/core/network/protocol_mocks.hpp" + namespace kagome::network { /** * Router, which reads and delivers different network messages to the @@ -26,6 +28,9 @@ namespace kagome::network { getFetchAvailableDataProtocol, (), (const, override)); + auto getMockedFetchAvailableDataProtocol() const { + return fetch_available_data; + } MOCK_METHOD(std::shared_ptr, getValidationProtocol, @@ -66,11 +71,17 @@ namespace kagome::network { getFetchChunkProtocol, (), (const, override)); + auto getMockedFetchChunkProtocol() const { + return fetch_chunk; + } MOCK_METHOD(std::shared_ptr, getFetchChunkProtocolObsolete, (), (const, override)); + auto getMockedFetchChunkProtocolObsolete() const { + return fetch_chunk_obsolete; + } MOCK_METHOD(std::shared_ptr, getFetchAttestedCandidateProtocol, @@ -82,6 +93,11 @@ namespace kagome::network { (), (const, override)); + MOCK_METHOD(std::shared_ptr, + getWarpProtocol, + (), + (const, override)); + MOCK_METHOD(std::shared_ptr, getStateProtocol, (), @@ -107,9 +123,62 @@ namespace kagome::network { (), (const, override)); - MOCK_METHOD(std::shared_ptr, - getPingProtocol, - (), - (const, override)); + MOCK_METHOD(std::shared_ptr, getPingProtocol, (), (const, override)); + + void setReturningMockedProtocols() { + // clang-format off + // ON_CALL(*this, getBlockAnnounceProtocol()).WillByDefault(testing::Return(block_announce)); + // ON_CALL(*this, getGrandpaProtocol()).WillByDefault(testing::Return(grandpa)); + // ON_CALL(*this, getSyncProtocol()).WillByDefault(testing::Return(sync)); + // ON_CALL(*this, getStateProtocol()).WillByDefault(testing::Return(state)); + // ON_CALL(*this, getWarpProtocol()).WillByDefault(testing::Return(warp)); + // ON_CALL(*this, getBeefyProtocol()).WillByDefault(testing::Return(beefy)); + // ON_CALL(*this, getBeefyJustificationProtocol()).WillByDefault(testing::Return(beefy_justifications)); + // ON_CALL(*this, getLightProtocol()).WillByDefault(testing::Return(light)); + // ON_CALL(*this, getPropagateTransactionsProtocol()).WillByDefault(testing::Return(propagate_transactions)); + // ON_CALL(*this, getValidationProtocol()).WillByDefault(testing::Return(validation)); + // ON_CALL(*this, getCollationProtocol()).WillByDefault(testing::Return(collation)); + // ON_CALL(*this, getCollationProtocolVStaging()).WillByDefault(testing::Return(collation_vstaging)); + // ON_CALL(*this, getValidationProtocolVStaging()).WillByDefault(testing::Return(validation_vstaging)); + // ON_CALL(*this, getReqCollationProtocol()).WillByDefault(testing::Return(req_collation)); + // ON_CALL(*this, getReqPovProtocol()).WillByDefault(testing::Return(req_pov)); + fetch_chunk = std::make_shared(); + ON_CALL(*this, getFetchChunkProtocol()).WillByDefault(testing::Return(fetch_chunk)); + fetch_chunk_obsolete = std::make_shared(); + ON_CALL(*this, getFetchChunkProtocolObsolete()).WillByDefault(testing::Return(fetch_chunk_obsolete)); + fetch_available_data = std::make_shared(); + ON_CALL(*this, getFetchAvailableDataProtocol()).WillByDefault(testing::Return(fetch_available_data)); + // ON_CALL(*this, getStatementFetchingProtocol()).WillByDefault(testing::Return(statement_fetching)); + // ON_CALL(*this, getSendDisputeProtocol()).WillByDefault(testing::Return(send_dispute)); + // ON_CALL(*this, getPing()).WillByDefault(testing::Return(ping)); + // ON_CALL(*this, getFetchAttestedCandidateProtocol()).WillByDefault(testing::Return(fetch_attested_candidate)); + // clang-format on + } + + private: + // clang-format off + // std::shared_ptr block_announce; + // std::shared_ptr grandpa; + // std::shared_ptr sync; + // std::shared_ptr state; + // std::shared_ptr warp; + // std::shared_ptr beefy; + // std::shared_ptr beefy_justifications; + // std::shared_ptr light; + // std::shared_ptr propagate_transactions; + // std::shared_ptr validation; + // std::shared_ptr collation; + // std::shared_ptr collation_vstaging; + // std::shared_ptr validation_vstaging; + // std::shared_ptr req_collation; + // std::shared_ptr req_pov; + std::shared_ptr fetch_chunk; + std::shared_ptr fetch_chunk_obsolete; + std::shared_ptr fetch_available_data; + // std::shared_ptr statement_fetching; + // std::shared_ptr send_dispute; + // std::shared_ptr ping; + // std::shared_ptr fetch_attested_candidate; + // clang-format on }; } // namespace kagome::network diff --git a/test/mock/core/parachain/availability_store_mock.hpp b/test/mock/core/parachain/availability_store_mock.hpp new file mode 100644 index 0000000000..4ab6d82cb2 --- /dev/null +++ b/test/mock/core/parachain/availability_store_mock.hpp @@ -0,0 +1,76 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "parachain/availability/store/store.hpp" + +#include + +namespace kagome::parachain { + + class AvailabilityStoreMock : public AvailabilityStore { + public: + MOCK_METHOD(bool, + hasChunk, + (const CandidateHash &candidate_hash, ChunkIndex index), + (const, override)); + + MOCK_METHOD(bool, + hasPov, + (const CandidateHash &candidate_hash), + (const, override)); + + MOCK_METHOD(bool, + hasData, + (const CandidateHash &candidate_hash), + (const, override)); + + MOCK_METHOD(std::optional, + getChunk, + (const CandidateHash &candidate_hash, ChunkIndex index), + (const, override)); + + MOCK_METHOD(std::optional, + getPov, + (const CandidateHash &candidate_hash), + (const, override)); + + MOCK_METHOD(std::optional, + getPovAndData, + (const CandidateHash &candidate_hash), + (const, override)); + + MOCK_METHOD(std::vector, + getChunks, + (const CandidateHash &candidate_hash), + (const, override)); + + MOCK_METHOD(void, + storeData, + (const network::RelayHash &, + const CandidateHash &, + std::vector &&, + const ParachainBlock &, + const PersistedValidationData &), + (override)); + + MOCK_METHOD(void, + putChunk, + (const network::RelayHash &, + const CandidateHash &, + ErasureChunk &&), + (override)); + + MOCK_METHOD(void, + remove, + (const network::RelayHash &relay_parent), + (override)); + + MOCK_METHOD(void, printStoragesLoad, (), (override)); + }; + +} // namespace kagome::parachain diff --git a/test/testutil/testparam.hpp b/test/testutil/testparam.hpp index 5b6f168183..d105241b7a 100644 --- a/test/testutil/testparam.hpp +++ b/test/testutil/testparam.hpp @@ -22,5 +22,5 @@ namespace testutil { bool should_fail, T &&value) { return {buffer, should_fail, std::forward(value)}; - }; + } } // namespace testutil