Feature: systematic chunks recovery (#2177)

* fix: clang-19 warns Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * format: remove semicolon Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * format: adjust by clang-format-16 Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * refactor: warpsync to router Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * refactor: keep discovered address longer Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * refactor: use only peer_id in protocols Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * refactor: separate RequestResponseProtocol to interface and implementation Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * refactor: separate some protocols' classes to interface and implementation Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: protocols' mocks Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: convenient router mock Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * update: erasure coding with systematic chunks reconstruction support Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: systematic chunks recovery Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * retrieve core index Signed-off-by: iceseer <[email protected]> * feature: obtaining backing_group for recovery in disputes Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: recovery test Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: review issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: self-review issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: CI issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * Fix recovery test Signed-off-by: Igor Egorov <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: remove deprecated clang-tidy rule Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: logs for recovery strategies Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * feature: logs for recovery strategies Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * hotfixe: clang-tidy Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * update clang version in dockerfile * ci - change container for clang tidy step * test new tidy workflow * llvm 19 * refactor: add some nolint suppression Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> * fix: clang-tidy issues Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> --------- Signed-off-by: Dmitriy Khaustov aka xDimon <[email protected]> Co-authored-by: iceseer <[email protected]> Co-authored-by: Igor Egorov <[email protected]> Co-authored-by: Kirill Azovtsev <[email protected]>
qdrvm · Aug 22, 2024 · 64162f3 · 64162f3
1 parent b0e47a5
commit 64162f3
Show file tree

Hide file tree

Showing 121 changed files with 1,993 additions and 613 deletions.
diff --git a/.clang-format b/.clang-format
@@ -22,7 +22,7 @@ QualifierAlignment: Custom
 QualifierOrder: ['inline', 'static', 'constexpr', 'const', 'volatile', 'type', 'restrict']
 ReferenceAlignment: Right
 ReflowComments: true
-RemoveSemicolon: true
+#RemoveSemicolon: true
 InsertNewlineAtEOF: true
 
 
diff --git a/.clang-tidy b/.clang-tidy
@@ -8,7 +8,6 @@
 Checks:          '-*,clang-analyzer-*,clang-diagnostic-*,readability-*,modernize-*,boost-*,bugprone-*,cppcoreguidelines-*,google-*,hicpp-*,performance-*,readability-*,-google-readability-namespace-comments,-readability-inconsistent-declaration-parameter-name,-readability-braces-around-statements,-hicpp-signed-bitwise,-google-runtime-references,-cppcoreguidelines-avoid-magic-numbers,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-readability-magic-numbers,-hicpp-explicit-conversions,-hicpp-uppercase-literal-suffix,-readability-uppercase-literal-suffix,-hicpp-no-array-decay,-hicpp-special-member-functions,-bugprone-narrowing-conversions,-modernize-use-nodiscard,-google-readability-braces-around-statements,-hicpp-braces-around-statements,-bugprone-suspicious-semicolon,-readability-named-parameter,-hicpp-named-parameter,-readability-identifier-naming,-modernize-use-trailing-return-type'
 WarningsAsErrors: 'modernize-*,cppcoreguidelines-*,boost-*,performance-*,google-build-using-namespace,readability-else-after-return,google-readability-todo'
 HeaderFilterRegex: '\.(hpp|h)'
-AnalyzeTemporaryDtors: false
 FormatStyle:     .clang-format
 User:            bogdan
 CheckOptions:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -43,7 +43,6 @@ env:
   DEFAULT_BUILD_TYPE: Release
   GIT_REF_NAME: ${{ github.ref_name }}
   IS_MAIN_OR_TAG: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) }}
-  CI: true
 
 
 jobs:
@@ -132,29 +131,6 @@ jobs:
       - name: "${{ matrix.options.name }}"
         run: "${{ matrix.options.run }}"
 
-  clang-tidy:
-    name: "Linux: clang-tidy"
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    container: qdrvm/kagome-dev:9-minideb
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: actions/cache@v4
-        with:
-          path: ${{ env.CACHE_PATHS }}
-          key: ${{ github.job }}-${{ env.CACHE_VERSION }}
-      - name: clang-tidy
-        env:
-          # build only generated files, so clang-tidy will work correctly
-          BUILD_FINAL_TARGET: generated
-        # run build for both WAVM and WasmEdge to download both, otherwise clang-tidy
-        # isn't able to find headers
-        run: |
-          ./housekeeping/make_build.sh 
-          ./housekeeping/clang-tidy-diff.sh
-
   coverage-self-hosted:
     if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted')  }}
     name: "Self-hosted: Linux: gcc-12 coverage/sonar"
@@ -218,6 +194,64 @@ jobs:
           BUILD_TYPE: "${{ matrix.options.build-type }}"
         run: ./housekeeping/docker/kagome-dev/make.sh
 
+  kagome_dev_docker_build_tidy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    name: "Linux: clang-tidy"
+
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+
+      - name: "Authenticate with Google Cloud"
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+
+      - name: "Set up Cloud SDK"
+        uses: 'google-github-actions/setup-gcloud@v2'
+
+      - name: "Configure Docker for GCR"
+        run: |
+          gcloud auth configure-docker --quiet
+          gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
+
+      - name: "Check version"
+        working-directory: ./housekeeping/docker/kagome-dev
+        run: |
+          SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2)
+          echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV
+
+      - name: "Cache dependencies"
+        id: cache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-${{ env.short_commit_hash }}
+          restore-keys: |
+            ${{ github.job }}-${{ env.CACHE_VERSION }}-tidy-
+
+      - name: "Build target"
+        working-directory: ./housekeeping/docker/kagome-dev
+        run:
+          make kagome_dev_docker_build_tidy \
+          DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
+          GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \
+          GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \
+          CI="true"
+
+      - name: "Cleaning cache"
+        run: |
+          find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \;
+
+      - name: "Always Save Cache"
+        id: cache-save
+        if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True')
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+
   kagome_dev_docker_build:
     runs-on: [ actions-runner-controller ]
     timeout-minutes: 180

diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake
@@ -77,8 +77,8 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM")
       LLVM
       VERSION 12.0.1-p4
       CMAKE_ARGS
-      LLVM_ENABLE_PROJECTS=ir
-      KEEP_PACKAGE_SOURCES
+        LLVM_ENABLE_PROJECTS=ir
+        KEEP_PACKAGE_SOURCES
   )
 
   if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -91,7 +91,7 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM")
       wavm
       VERSION 1.0.14
       CMAKE_ARGS
-      WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS}
+        WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS}
       KEEP_PACKAGE_SOURCES
   )
 endif ()
@@ -106,5 +106,14 @@ hunter_config(
     libsecp256k1
     VERSION 0.5.1
     CMAKE_ARGS
-        SECP256K1_ENABLE_MODULE_RECOVERY=ON
+      SECP256K1_ENABLE_MODULE_RECOVERY=ON
+)
+
+hunter_config(
+    erasure_coding_crust
+#    VERSION 0.0.8
+    URL  https://github.com/qdrvm/erasure-coding-crust/archive/refs/tags/v0.0.8.tar.gz
+    SHA1 6bcdb6327f5da2dcec5c70f2fa63b95a44925af0
+    KEEP_PACKAGE_SOURCES
 )
+
diff --git a/cmake/clang-tidy.cmake b/cmake/clang-tidy.cmake
@@ -1,6 +1,6 @@
 if(NOT CLANG_TIDY_BIN)
   find_program(CLANG_TIDY_BIN
-    NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7
+    NAMES clang-tidy clang-tidy-20 clang-tidy-19 clang-tidy-18 clang-tidy-17 clang-tidy-16
     DOC "Path to clang-tidy executable"
   )
 endif()

diff --git a/core/api/service/author/requests/has_key.hpp b/core/api/service/author/requests/has_key.hpp
@@ -20,7 +20,7 @@ namespace kagome::api::author::request {
    public:
     explicit HasKey(std::shared_ptr<AuthorApi> api) : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<Return> execute() override {
       OUTCOME_TRY(public_key, common::unhexWith0x(getParam<0>()));

diff --git a/core/api/service/author/requests/has_session_keys.hpp b/core/api/service/author/requests/has_session_keys.hpp
@@ -19,7 +19,7 @@ namespace kagome::api::author::request {
     explicit HasSessionKeys(std::shared_ptr<AuthorApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<Return> execute() override {
       OUTCOME_TRY(keys, common::unhexWith0x(getParam<0>()));

diff --git a/core/api/service/author/requests/insert_key.hpp b/core/api/service/author/requests/insert_key.hpp
@@ -25,7 +25,7 @@ namespace kagome::api::author::request {
    public:
     explicit InsertKey(std::shared_ptr<AuthorApi> api) : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<Return> execute() override {
       auto &seed_hex = getParam<1>();

diff --git a/core/api/service/author/requests/rotate_keys.hpp b/core/api/service/author/requests/rotate_keys.hpp
@@ -18,7 +18,7 @@ namespace kagome::api::author::request {
    public:
     explicit RotateKeys(std::shared_ptr<AuthorApi> api) : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<Return> execute() override {
       return api_->rotateKeys();

diff --git a/core/api/service/author/requests/submit_and_watch_extrinsic.hpp b/core/api/service/author/requests/submit_and_watch_extrinsic.hpp
@@ -9,6 +9,7 @@
 #include <jsonrpc-lean/request.h>
 
 #include "api/service/author/author_api.hpp"
+#include "api/service/base_request.hpp"
 #include "primitives/extrinsic.hpp"
 
 namespace kagome::api::author::request {
@@ -19,7 +20,7 @@ namespace kagome::api::author::request {
     explicit SubmitAndWatchExtrinsic(std::shared_ptr<AuthorApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<primitives::SubscriptionId> execute() override {
       auto ext_hex = getParam<0>();

diff --git a/core/api/service/author/requests/submit_extrinsic.hpp b/core/api/service/author/requests/submit_extrinsic.hpp
@@ -9,6 +9,7 @@
 #include <jsonrpc-lean/request.h>
 
 #include "api/service/author/author_api.hpp"
+#include "api/service/base_request.hpp"
 #include "primitives/extrinsic.hpp"
 
 namespace kagome::api::author::request {
@@ -19,7 +20,7 @@ namespace kagome::api::author::request {
     explicit SubmitExtrinsic(std::shared_ptr<AuthorApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<common::Hash256> execute() override {
       auto ext_hex = getParam<0>();

diff --git a/core/api/service/chain/requests/get_block_hash.cpp b/core/api/service/chain/requests/get_block_hash.cpp
@@ -89,6 +89,6 @@ namespace kagome::api::chain::request {
 
           return results;
         });
-  };
+  }
 
 }  // namespace kagome::api::chain::request
diff --git a/core/api/service/child_state/requests/get_keys.hpp b/core/api/service/child_state/requests/get_keys.hpp
@@ -30,7 +30,7 @@ namespace kagome::api::child_state::request {
     explicit GetKeys(std::shared_ptr<ChildStateApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
     ~GetKeys() = default;
 
     outcome::result<void> init(const jsonrpc::Request::Parameters &params);

diff --git a/core/api/service/child_state/requests/get_keys_paged.hpp b/core/api/service/child_state/requests/get_keys_paged.hpp
@@ -30,7 +30,7 @@ namespace kagome::api::child_state::request {
     explicit GetKeysPaged(std::shared_ptr<ChildStateApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
     ~GetKeysPaged() = default;
 
     outcome::result<void> init(const jsonrpc::Request::Parameters &params);

diff --git a/core/api/service/rpc/requests/methods.hpp b/core/api/service/rpc/requests/methods.hpp
@@ -32,7 +32,7 @@ namespace kagome::api::rpc::request {
 
     explicit Methods(std::shared_ptr<RpcApi> api) : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
     ~Methods() = default;
 
     outcome::result<void> init(const jsonrpc::Request::Parameters &params);

diff --git a/core/api/service/state/requests/call.cpp b/core/api/service/state/requests/call.cpp
@@ -12,7 +12,7 @@ namespace kagome::api::state::request {
 
   Call::Call(std::shared_ptr<StateApi> api) : api_(std::move(api)) {
     BOOST_ASSERT(api_);
-  };
+  }
 
   outcome::result<void> Call::init(const jsonrpc::Request::Parameters &params) {
     if (params.size() < 2 or params.size() > 3) {

diff --git a/core/api/service/state/requests/get_keys_paged.hpp b/core/api/service/state/requests/get_keys_paged.hpp
@@ -30,7 +30,7 @@ namespace kagome::api::state::request {
     explicit GetKeysPaged(std::shared_ptr<StateApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
     ~GetKeysPaged() = default;
 
     outcome::result<void> init(const jsonrpc::Request::Parameters &params);

diff --git a/core/api/service/state/requests/query_storage.hpp b/core/api/service/state/requests/query_storage.hpp
@@ -44,7 +44,7 @@ namespace kagome::api::state::request {
     explicit QueryStorage(std::shared_ptr<StateApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<std::vector<StateApi::StorageChangeSet>> execute()
         override {

diff --git a/core/api/service/system/requests/account_next_index.hpp b/core/api/service/system/requests/account_next_index.hpp
@@ -17,7 +17,7 @@ namespace kagome::api::system::request {
     explicit AccountNextIndex(std::shared_ptr<SystemApi> api)
         : api_(std::move(api)) {
       BOOST_ASSERT(api_);
-    };
+    }
 
     outcome::result<primitives::AccountNonce> execute() override {
       return api_->getNonceFor(getParam<0>());

diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp
@@ -211,8 +211,8 @@ namespace kagome::application {
      * List of telemetry endpoints specified via CLI argument or config file
      * @return a vector of parsed telemetry endpoints
      */
-    virtual const std::vector<telemetry::TelemetryEndpoint>
-        &telemetryEndpoints() const = 0;
+    virtual const std::vector<telemetry::TelemetryEndpoint> &
+    telemetryEndpoints() const = 0;
 
     /**
      * @return enum constant of the chosen sync method

diff --git a/core/application/chain_spec.hpp b/core/application/chain_spec.hpp
@@ -38,8 +38,8 @@ namespace kagome::application {
     virtual const std::vector<libp2p::multi::Multiaddress> &bootNodes()
         const = 0;
 
-    virtual const std::vector<std::pair<std::string, size_t>>
-        &telemetryEndpoints() const = 0;
+    virtual const std::vector<std::pair<std::string, size_t>> &
+    telemetryEndpoints() const = 0;
 
     virtual const std::string &protocolId() const = 0;
 

diff --git a/core/authority_discovery/query/query_impl.cpp b/core/authority_discovery/query/query_impl.cpp
@@ -283,7 +283,7 @@ namespace kagome::authority_discovery {
     }
 
     std::ignore = host_.getPeerRepository().getAddressRepository().addAddresses(
-        peer.id, peer.addresses, libp2p::peer::ttl::kRecentlyConnected);
+        peer.id, peer.addresses, libp2p::peer::ttl::kDay);
 
     peer_to_auth_cache_.insert_or_assign(peer.id, authority);
     auth_to_peer_cache_.insert_or_assign(

diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp
@@ -425,18 +425,18 @@ namespace kagome::blockchain {
       std::shared_ptr<storage::trie_pruner::TriePruner> state_pruner,
       common::MainThreadPool &main_thread_pool)
       : block_tree_data_{BlockTreeData{
-            .header_repo_ = std::move(header_repo),
-            .storage_ = std::move(storage),
-            .state_pruner_ = std::move(state_pruner),
-            .tree_ = std::make_unique<CachedTree>(finalized),
-            .extrinsic_observer_ = std::move(extrinsic_observer),
-            .hasher_ = std::move(hasher),
-            .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo),
-            .justification_storage_policy_ =
-                std::move(justification_storage_policy),
-            .genesis_block_hash_ = {},
-            .blocks_pruning_ = {app_config.blocksPruning(), finalized.number},
-        }},
+          .header_repo_ = std::move(header_repo),
+          .storage_ = std::move(storage),
+          .state_pruner_ = std::move(state_pruner),
+          .tree_ = std::make_unique<CachedTree>(finalized),
+          .extrinsic_observer_ = std::move(extrinsic_observer),
+          .hasher_ = std::move(hasher),
+          .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo),
+          .justification_storage_policy_ =
+              std::move(justification_storage_policy),
+          .genesis_block_hash_ = {},
+          .blocks_pruning_ = {app_config.blocksPruning(), finalized.number},
+      }},
         main_pool_handler_{main_thread_pool.handlerStarted()} {
     block_tree_data_.sharedAccess([&](const BlockTreeData &p) {
       BOOST_ASSERT(p.header_repo_ != nullptr);

diff --git a/core/common/variant_builder.hpp b/core/common/variant_builder.hpp
@@ -82,7 +82,7 @@ namespace kagome::common {
       static_assert(dynamic_variant::is_boost_variant<Variant>::value);
       dynamic_variant::functors_vector_builder<Variant> builder = {&funcs_};
       boost::mpl::for_each<Types>(builder);
-    };
+    }
 
     /**
      * Initializes the referenced variant with default constructed instance of

diff --git a/core/consensus/grandpa/impl/voting_round_impl.hpp b/core/consensus/grandpa/impl/voting_round_impl.hpp
@@ -167,7 +167,7 @@ namespace kagome::consensus::grandpa {
      */
     std::shared_ptr<VotingRound> getPreviousRound() const override {
       return previous_round_;
-    };
+    }
 
     /**
      * Removes previous round to limit chain of rounds
@@ -226,7 +226,7 @@ namespace kagome::consensus::grandpa {
      */
     const std::optional<BlockInfo> &finalizedBlock() const override {
       return finalized_;
-    };
+    }
 
     /**
      * @return state containing round number, last finalized block, votes, and