diff --git a/CMakeLists.txt b/CMakeLists.txt index 29e5097e6..9dfef91c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,9 @@ if(ENABLE_ASSERTIONS) endif() if(FIZZY_FUZZING) - set(fuzzing_flags -fsanitize=fuzzer-no-link,address,undefined,nullability,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation) + if(NOT CMAKE_BUILD_TYPE STREQUAL Coverage) + set(fuzzing_flags -fsanitize=fuzzer-no-link,address,undefined,nullability,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation) + endif() add_compile_options(${fuzzing_flags}) add_link_options(${fuzzing_flags}) endif() diff --git a/cmake/ProjectWabt.cmake b/cmake/ProjectWabt.cmake index bd16b6ef2..d522d4e8a 100644 --- a/cmake/ProjectWabt.cmake +++ b/cmake/ProjectWabt.cmake @@ -11,7 +11,7 @@ set(binary_dir ${prefix}/src/wabt-build) set(include_dir ${source_dir}) set(wabt_library ${binary_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}wabt${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(flags -fvisibility=hidden) +set(flags "${fuzzing_flags} -fvisibility=hidden") if(SANITIZE MATCHES address) # Instrument WABT with ASan - required for container-overflow checks. set(flags "-fsanitize=address ${flags}") @@ -24,12 +24,12 @@ endif() ExternalProject_Add(wabt EXCLUDE_FROM_ALL 1 PREFIX ${prefix} - DOWNLOAD_NAME wabt-1.0.19.tar.gz + DOWNLOAD_NAME wabt-fixes.tar.gz DOWNLOAD_DIR ${prefix}/downloads SOURCE_DIR ${source_dir} BINARY_DIR ${binary_dir} - URL https://github.com/WebAssembly/wabt/archive/1.0.19.tar.gz - URL_HASH SHA256=134f2afc8205d0a3ab89c5f0d424ff3823e9d2769c39d2235aa37eba7abc15ba + URL https://github.com/ewasm/wabt/archive/fixes.tar.gz + URL_HASH SHA256=2df5d7aef95197e7c207935ff9318524b69f41458c1473a857f4771e8ec75e4c CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/cmake/spectests_contributing.md b/cmake/spectests_contributing.md new file mode 100644 index 000000000..29bee7b48 --- /dev/null +++ b/cmake/spectests_contributing.md @@ -0,0 +1,145 @@ + + +## Run fuzzer + +Run Fizzy vs WABT verification fuzzer, currently in `fuzzing` branch. +Start with small inputs. + +```shell script +bin/fizzy-fuzz-parser corpus -max_len=13 +``` + +In case it finds a failure you will see output like +```text +INFO: Seed: 770491456 +INFO: Loaded 1 modules (64068 inline 8-bit counters): 64068 [0xb8dc90, 0xb9d6d4), +INFO: Loaded 1 PC tables (64068 PCs): 64068 [0xb9d6d8,0xc97b18), +INFO: 10986 files found in corpus +INFO: seed corpus: files: 10986 min: 1b max: 27419b total: 740862b rss: 40Mb +#11160 INITED cov: 7782 ft: 9644 corp: 227/2677b exec/s: 0 rss: 186Mb +#65536 pulse cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 215Mb +#131072 pulse cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 248Mb +#262144 pulse cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 314Mb + MALFORMED: invalid limits 4 +/home/chfast/Projects/wasmx/fizzy/test/fuzzer/parser_fuzzer.cpp:47:9: runtime error: execution reached an unreachable program point +SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/chfast/Projects/wasmx/fizzy/test/fuzzer/parser_fuzzer.cpp:47:9 in +MS: 1 ChangeBit-; base unit: 846f4928f92649789961543e93544b4c2ed6fc44 +0x0,0x61,0x73,0x6d,0x1,0x0,0x0,0x0,0x5,0x3,0x1,0x4,0x2b, +\x00asm\x01\x00\x00\x00\x05\x03\x01\x04+ +artifact_prefix='./'; Test unit written to ./crash-016934f781c41276bfacda9a90385cef85a88d5f +Base64: AGFzbQEAAAAFAwEEKw== +``` + +In this example Fizzy reported the error `MALFORMED: invalid limits 4` +while WABT considers the Wasm binary valid. +The test unit (Wasm binary) is also printed in various forms and saved as +`crash-016934f781c41276bfacda9a90385cef85a88d5f`. + + +## Check WebAssembly spec interpreter + +In case of false negative validation result in WABT it is likely that +such case is missing in the WAST spec test. + +Build the `wasm reference interpreter` following official instructions. + +Rename `crash-016934f781c41276bfacda9a90385cef85a88d5f` to `crash.wasm` +as the interpreter need proper files extension to recognize file type. + +```shell script +./wasm crash.wasm +``` + +Output: +```text +crash.wasm:0xb: decoding error: integer too large +``` + +Good news. This is effectively the same error reported as in Fizzy. In Wasm 1.0 `limits` kind +can be 0 or 1. The the value 4 in the `crash.wasm` is larger than the max 1. + + +## Inspect with WABT tools + +You can use `wasm2wat` or `wasm-validate` tools to confirm the bug in WABT. +Make sure you use the same version as used for fuzzing. +Disable all extensions except "mutable globals". + +In our example, `wasm2wat crash.wasm`: +```webassembly +(module + (memory (;0;) 43 i64)) +``` +This indicates that the issue is in the memory section. + +Unfortunately, we get different error as the data section also has invalid length. +Let's fix that. + + +## Prepare WAST test + +Dump the test unit (`xxd -p -c1000 crash.wasm`): + +```hexdump +0061736d01000000050301042b +``` + +Here are created WAST tests, placed in `new.wast` file. In the original `crash.wasm` the invalid 4 +is followed by `2b` byte. We added 2 more variants of the test with byte `00` and without any +following byte (remember to change the section length if you actually modify the length). + +```wast +(assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\05\03\01" ;; memory section + "\04\2b" ;; malformed memory limit 4 + ) + "integer too large" +) + +(assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\05\03\01" ;; memory section + "\04\00" ;; malformed memory limit 4 + ) + "integer too large" +) + +(assert_malformed + (module binary + "\00asm" "\01\00\00\00" + "\05\02\01" ;; memory section + "\04" ;; malformed memory limit 4 + ) + "integer too large" +) +``` + +These tests can be rechecked with WABT. + +```shell script +wast2json new.wast +wasm-validate new.0.wasm +wasm-validate new.1.wasm +wasm-validate new.2.wasm +``` + +The last test case is actually failing in WABT for some other reason, but it can stay as a valid +spectests addition. + +```text +000000c: error: unable to read u32 leb128: memory initial page count +``` + + +## Debug & fix WABT + +Debug `wasm-validate new.0.wasm` to find out why it passes validation. +Binary loading code is in `binary-reader.cc`. + +The example bug is fixed by https://github.com/WebAssembly/wabt/pull/1547. + + +## Send WAST tests upstream \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3025f313c..68d3bca1b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,6 +22,9 @@ if(HUNTER_ENABLED) endif() find_package(GTest REQUIRED) +include(ProjectWabt) +include(ProjectWasm3) + add_subdirectory(utils) add_subdirectory(bench) add_subdirectory(bench_internal) diff --git a/test/fuzzer/CMakeLists.txt b/test/fuzzer/CMakeLists.txt index a5570b54b..feda1539c 100644 --- a/test/fuzzer/CMakeLists.txt +++ b/test/fuzzer/CMakeLists.txt @@ -4,4 +4,4 @@ add_executable(fizzy-fuzz-parser parser_fuzzer.cpp) target_link_options(fizzy-fuzz-parser PRIVATE -fsanitize=fuzzer) -target_link_libraries(fizzy-fuzz-parser PRIVATE fizzy::fizzy-internal) +target_link_libraries(fizzy-fuzz-parser PRIVATE fizzy::fizzy-internal wabt::wabt) diff --git a/test/fuzzer/parser_fuzzer.cpp b/test/fuzzer/parser_fuzzer.cpp index 614bc5957..6fe32f71d 100644 --- a/test/fuzzer/parser_fuzzer.cpp +++ b/test/fuzzer/parser_fuzzer.cpp @@ -3,18 +3,175 @@ // SPDX-License-Identifier: Apache-2.0 #include "parser.hpp" +#include +#include +#include -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t data_size) noexcept +#include +#include +#include +#include + +namespace +{ +struct Stats +{ + int64_t malformed = 0; + int64_t invalid = 0; + int64_t valid = 0; + + ~Stats() + { + const auto all = malformed + invalid + valid; + if (all == 0) + return; + std::clog << "WASM STATS" << std::setprecision(3) << "\n all: " << all + << "\n malformed: " << malformed << " " << ((malformed * 100) / all) << "%" + << "\n invalid: " << invalid << " " << ((invalid * 100) / all) << "%" + << "\n valid: " << valid << " " << ((valid * 100) / all) << "%" + << "\n"; + } +}; + +Stats stats; + +void handle_unexpected_errors() noexcept +{ + static const bool ignore_errors = [] { + const auto options = std::getenv("OPTIONS"); + if (!options) + return false; + return std::string{options}.find("ignore_errors") != std::string::npos; + }(); + if (!ignore_errors) + __builtin_unreachable(); +} + +//constexpr auto wabt_ignored_errors = { +// "unable to read u32 leb128: version", +// "invalid linking metadata version:", +//}; + +wabt::Errors wabt_errors; + +bool wabt_parse(const uint8_t* data, size_t data_size) noexcept +{ + using namespace wabt; + + ReadBinaryOptions read_options; + read_options.features.enable_mutable_globals(); + read_options.features.disable_exceptions(); + read_options.features.disable_annotations(); + read_options.features.disable_bulk_memory(); + read_options.features.disable_gc(); + read_options.features.disable_memory64(); + read_options.features.disable_multi_value(); + read_options.features.disable_reference_types(); + read_options.features.disable_sat_float_to_int(); + read_options.features.disable_sign_extension(); + read_options.features.disable_simd(); + read_options.features.disable_tail_call(); + read_options.features.disable_threads(); + read_options.fail_on_custom_section_error = false; + read_options.stop_on_first_error = true; + read_options.read_debug_names = false; + Module module; + + wabt_errors.clear(); + + { + const auto result = + ReadBinaryIr("fuzzing", data, data_size, read_options, &wabt_errors, &module); + + if (Failed(result)) + return false; + + wabt_errors.clear(); // Clear errors (probably) from custom sections. + } + + { + const auto result = + ValidateModule(&module, &wabt_errors, ValidateOptions{read_options.features}); + if (Failed(result)) + return false; + } + + return wabt_errors.empty(); +} + +} // namespace + +extern "C" { + +size_t LLVMFuzzerMutate(uint8_t* data, size_t size, size_t max_size) noexcept; + +size_t LLVMFuzzerCustomMutator( + uint8_t* data, size_t size, size_t max_size, [[maybe_unused]] unsigned int seed) noexcept { + static constexpr uint8_t wasm_prefix[]{0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00}; + static constexpr auto wasm_prefix_size = sizeof(wasm_prefix); + + // For inputs shorter than wasm prefix just mutate it. + if (size <= wasm_prefix_size) + return LLVMFuzzerMutate(data, size, max_size); + + // For other, leave prefix unchanged. It is likely to be valid and we don't want to waste time + // on mutating the prefix. + const auto new_size_without_prefix = LLVMFuzzerMutate( + data + wasm_prefix_size, size - wasm_prefix_size, max_size - wasm_prefix_size); + return new_size_without_prefix + wasm_prefix_size; +} + +int LLVMFuzzerTestOneInput(const uint8_t* data, size_t data_size) noexcept +{ + const auto expected = wabt_parse(data, data_size); + try { fizzy::parse({data, data_size}); + ++stats.valid; + if (!expected) + { + bool has_errors = false; + for (const auto& err : wabt_errors) + { +// bool ignored = false; +// +// for (const auto& m : wabt_ignored_errors) +// { +// if (err.message.find(m) != std::string::npos) +// ignored = true; +// } +// if (ignored) +// continue; + + std::cerr << " MISSED ERROR: " << err.message << "\n"; + has_errors = true; + } + + if (has_errors) + handle_unexpected_errors(); + } } - catch (const fizzy::parser_error&) + catch (const fizzy::parser_error& err) { + ++stats.malformed; + if (expected) + { + std::cerr << " MALFORMED: " << err.what() << "\n"; + handle_unexpected_errors(); + } } - catch (const fizzy::validation_error&) + catch (const fizzy::validation_error& err) { + ++stats.invalid; + if (expected) + { + std::cerr << " INVALID: " << err.what() << "\n"; + handle_unexpected_errors(); + } } + return 0; } +} diff --git a/test/utils/CMakeLists.txt b/test/utils/CMakeLists.txt index 16a3afc40..6b6c13a70 100644 --- a/test/utils/CMakeLists.txt +++ b/test/utils/CMakeLists.txt @@ -2,9 +2,6 @@ # Copyright 2019-2020 The Fizzy Authors. # SPDX-License-Identifier: Apache-2.0 -include(ProjectWabt) -include(ProjectWasm3) - add_library(test-utils STATIC) add_library(fizzy::test-utils ALIAS test-utils)