Merge branch 'master' into test/polkadot-functional-zombietests

.github/workflows/test.yml

@@ -32,7 +32,7 @@ env:
 jobs:
   MacOS:
     runs-on: macos-14
-    timeout-minutes: 60
+    timeout-minutes: 120
     steps:
     - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
     - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -54,13 +54,13 @@ jobs:
       fail-fast: false
       matrix:
         options:
-          - name: "Linux: clang-15 External Project"
-            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake
+          - name: "Linux: clang-16 External Project"
+            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake
 
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -82,46 +82,6 @@ jobs:
           du -hd1 /__w /github || true
 
   Linux-self-hosted:
-    if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' ) }}
-    strategy:
-      fail-fast: false
-      matrix:
-        options:
-          - name: "Self-hosted: Linux: gcc-12 ASAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON
-          - name: "Self-hosted: Linux: clang-15 TSAN WAVM"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
-          - name: "Self-hosted: Linux: clang-15 UBSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON
-          - name: "Self-hosted: Linux: clang-15 External Project"
-            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake
-
-    name: "${{ matrix.options.name }}"
-    runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
-    steps:
-      - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
-      - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
-        with:
-          path: ${{ env.CACHE_PATHS }}
-          key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
-      - name: df du
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
-      - name: Install mold
-        run: ./housekeeping/ci_install_mold.sh --make-default
-      - name: "${{ matrix.options.name }}"
-        run: "${{ matrix.options.run }}"
-      - name: df du
-        if: ${{ always() }}
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
-
-  Linux-self-hosted-PR:
-    if: ${{ contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }}
     strategy:
       fail-fast: false
       matrix:
@@ -130,40 +90,31 @@ jobs:
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON
           - name: "Self-hosted: Linux: gcc-12 TSAN WAVM"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
-          - name: "Self-hosted: Linux: clang-15 UBSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx20.cmake -DUBSAN=ON
-          - name: "Self-hosted: Linux: clang-15 External Project"
-            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-15_cxx20.cmake
+          - name: "Self-hosted: Linux: clang-16 UBSAN"
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON
+          - name: "Self-hosted: Linux: clang-16 External Project"
+            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake
 
     name: "${{ matrix.options.name }}"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
         with:
           path: ${{ env.CACHE_PATHS }}
           key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
-      - name: df du
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
       - name: Install mold
         run: ./housekeeping/ci_install_mold.sh --make-default
       - name: "${{ matrix.options.name }}"
         run: "${{ matrix.options.run }}"
-      - name: df du
-        if: ${{ always() }}
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
 
   clang-tidy:
     name: "Linux: clang-tidy"
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
         with:
@@ -186,29 +137,20 @@ jobs:
     if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted')  }}
     name: "Self-hosted: Linux: gcc-12 coverage/sonar"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
         with:
           path: ${{ env.CACHE_PATHS }}
           key: ${{ github.job }}-${{ env.CACHE_VERSION }}
-      - name: df du
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
       - name: Install mold
         run: ./housekeeping/ci_install_mold.sh --make-default
       - name: makeBuild
         env:
           BUILD_FINAL_TARGET: ctest_coverage
         run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON
-      - name: df du
-        if: ${{ always() }}
-        run: |
-          df -m || true
-          du -hd1 /__w /github || true
       - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
         name: Submit Coverage
         env:
@@ -235,8 +177,8 @@ jobs:
             build-type: "Release"
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -260,8 +202,8 @@ jobs:
             build-type: "Release"
     name: "${{ matrix.options.name }}"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev:7-minideb
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -287,8 +229,8 @@ jobs:
           - name: "Self-hosted: Push RelWithDebInfo Docker image"
             build-type: "RelWithDebInfo"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -325,8 +267,8 @@ jobs:
           - name: "Self-hosted: Push RelWithDebInfo Docker image"
             build-type: "RelWithDebInfo"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    timeout-minutes: 120
+    container: qdrvm/kagome-dev:8-minideb
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
         with:
@@ -381,7 +323,7 @@ jobs:
           - name: "Test we are producing blocks at 6 seconds clip"
             test: "test-polkadot-functional-0011-async-backing-6-seconds-rate"
     runs-on: [ actions-runner-controller ]
-    timeout-minutes: 60
+    timeout-minutes: 120
     needs: [Push-self-hosted]
     steps:
       - name: Set owner of working dir recurively

cmake/Hunter/config.cmake

@@ -40,7 +40,7 @@ hunter_config(
 
 hunter_config(
     libp2p
-    VERSION 0.1.21
+    VERSION 0.1.22
     KEEP_PACKAGE_SOURCES
 )
 

cmake/Hunter/hunter-gate-url.cmake

@@ -1,5 +1,5 @@
 HunterGate(
-  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm10.zip
-  SHA1 9571399d8d091420131eb81f884521326c9d3615
+  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm11.zip
+  SHA1 20620ee1e2e9738b2d4561be2622d71e8c73bee9
   LOCAL
 )

cmake/toolchain/clang-16_cxx20.cmake

@@ -0,0 +1,2 @@
+include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake)

cmake/toolchain/clang-15_mold_cxx20.cmake → cmake/toolchain/clang-16_mold_cxx20.cmake

@@ -1,3 +1,3 @@
-include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-15.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/linker/mold.cmake)

cmake/toolchain/compiler/clang-15.cmake → cmake/toolchain/compiler/clang-16.cmake

@@ -1,7 +1,7 @@
-if(DEFINED POLLY_COMPILER_CLANG_15_CMAKE)
+if(DEFINED POLLY_COMPILER_CLANG_16_CMAKE)
   return()
 else()
-  set(POLLY_COMPILER_CLANG_15_CMAKE 1)
+  set(POLLY_COMPILER_CLANG_16_CMAKE 1)
 endif()
 
 include(${CMAKE_CURRENT_LIST_DIR}/../../print.cmake)
@@ -13,29 +13,29 @@ if(XCODE_VERSION)
   fatal_error(${_err})
 endif()
 
-find_program(CMAKE_C_COMPILER clang-15 REQUIRED)
-find_program(CMAKE_CXX_COMPILER clang++-15)
+find_program(CMAKE_C_COMPILER clang-16 REQUIRED)
+find_program(CMAKE_CXX_COMPILER clang++-16)
 
 if (CMAKE_CXX_COMPILER STREQUAL "CMAKE_CXX_COMPILER-NOTFOUND")
-  message(STATUS "clang++-15 not found, checking clang++")
+  message(STATUS "clang++-16 not found, checking clang++")
   cmake_path(GET CMAKE_C_COMPILER PARENT_PATH compiler_path)
   message(STATUS "Assumed compiler path: ${compiler_path}")
-  # clang++-15 doesn't always exist
+  # clang++-16 doesn't always exist
   find_program(CMAKE_CXX_COMPILER clang++ PATHS "${compiler_path}" NO_DEFAULT_PATH REQUIRED)
 
   execute_process(COMMAND "${CMAKE_CXX_COMPILER}" --version OUTPUT_VARIABLE compiler_version_output)
   string(REGEX MATCH "clang version ([0-9]+)\\.[0-9]+\\.[0-9]+" compiler_version "${compiler_version_output}")
-  if (NOT CMAKE_MATCH_1 STREQUAL "15")
-    message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 15 is required")
+  if (NOT CMAKE_MATCH_1 STREQUAL "16")
+    message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 16 is required")
   endif()
 endif()
 
 if(NOT CMAKE_C_COMPILER)
-  fatal_error("clang-15 not found")
+  fatal_error("clang-16 not found")
 endif()
 
 if(NOT CMAKE_CXX_COMPILER)
-  fatal_error("clang++-15 not found")
+  fatal_error("clang++-16 not found")
 endif()
 
 set(

cmake/toolchain/flags/sanitize_thread.cmake

@@ -9,13 +9,22 @@ endif ()
 
 include(${CMAKE_CURRENT_LIST_DIR}/../../add_cache_flag.cmake)
 
+set(TSAN_IGNORELIST "${CMAKE_CURRENT_LIST_DIR}/../../../.thread-sanitizer-ignore")
+
+set(ENV{TSAN_OPTIONS} "suppressions=${TSAN_IGNORELIST}")
+
 set(FLAGS
     -fsanitize=thread
-    -fsanitize-blacklist="${CMAKE_CURRENT_LIST_DIR}/../../../.thread-sanitizer-ignore"
-    -fsanitize-ignorelist="${CMAKE_CURRENT_LIST_DIR}/../../../.thread-sanitizer-ignore"
     -g
     -O1
 )
+if (CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+  set(FLAGS
+      ${FLAGS}
+      -fsanitize-blacklist="${TSAN_IGNORELIST}"
+      -fsanitize-ignorelist="${TSAN_IGNORELIST}"
+  )
+endif()
 
 foreach(FLAG IN LISTS FLAGS)
   add_cache_flag(CMAKE_CXX_FLAGS ${FLAG})

core/application/app_configuration.hpp

@@ -264,6 +264,11 @@ namespace kagome::application {
      */
     virtual std::chrono::milliseconds pvfSubprocessDeadline() const = 0;
 
+    /**
+     * Whether secure validator mode should be disabled.
+     */
+    virtual bool disableSecureMode() const = 0;
+
     enum class OffchainWorkerMode { WhenValidating, Always, Never };
     /**
      * @return enum constant of the mode of run offchain workers

core/application/impl/app_configuration_impl.cpp

@@ -6,6 +6,7 @@
 
 #include "application/impl/app_configuration_impl.hpp"
 
+#include <boost/program_options/value_semantic.hpp>
 #include <charconv>
 #include <filesystem>
 #include <limits>
@@ -891,6 +892,7 @@ namespace kagome::application {
         "Disables spawn of child pvf check processes, thus they could not be aborted by deadline timer")
         ("parachain-check-deadline", po::value<uint32_t>()->default_value(2000),
         "Pvf check subprocess execution deadline in milliseconds")
+        ("insecure-validator-i-know-what-i-do", po::bool_switch(), "Allows a validator to run insecurely outside of Secure Validator Mode.")
         ;
     po::options_description benchmark_desc("Benchmark options");
     benchmark_desc.add_options()
@@ -1512,6 +1514,10 @@ namespace kagome::application {
       pvf_subprocess_deadline_ = std::chrono::milliseconds(*arg);
     }
 
+    if (find_argument(vm, "insecure-validator-i-know-what-i-do")) {
+      disable_secure_mode_ = true;
+    }
+
     bool offchain_worker_value_error = false;
     find_argument<std::string>(
         vm,

core/application/impl/app_configuration_impl.hpp

@@ -232,6 +232,9 @@ namespace kagome::application {
     std::chrono::milliseconds pvfSubprocessDeadline() const override {
       return pvf_subprocess_deadline_;
     }
+    bool disableSecureMode() const override {
+      return disable_secure_mode_;
+    }
 
    private:
     void parse_general_segment(const rapidjson::Value &val);
@@ -383,6 +386,7 @@ namespace kagome::application {
     bool should_precompile_parachain_modules_{true};
     bool use_pvf_subprocess_{true};
     std::chrono::milliseconds pvf_subprocess_deadline_{2000};
+    bool disable_secure_mode_{false};
   };
 
 }  // namespace kagome::application

core/common/blob.hpp

@@ -233,6 +233,10 @@ namespace kagome::common {
 
 }  // namespace kagome::common
 
+namespace kagome {
+  using common::Hash256;
+}  // namespace kagome
+
 template <size_t N>
 struct std::hash<kagome::common::Blob<N>> {
   auto operator()(const kagome::common::Blob<N> &blob) const {

core/common/buffer.hpp

@@ -239,6 +239,10 @@ namespace kagome::common {
 
 }  // namespace kagome::common
 
+namespace kagome {
+  using common::Buffer;
+}  // namespace kagome
+
 template <size_t N>
 struct std::hash<kagome::common::SLBuffer<N>> {
   size_t operator()(const kagome::common::SLBuffer<N> &x) const {

core/common/buffer_view.hpp

@@ -95,6 +95,10 @@ namespace kagome::common {
   }
 }  // namespace kagome::common
 
+namespace kagome {
+  using common::BufferView;
+}  // namespace kagome
+
 template <>
 struct fmt::formatter<kagome::common::BufferView> {
   // Presentation format: 's' - short, 'l' - long.