Validation fuzzing

CMakeLists.txt

@@ -83,7 +83,9 @@ if(ENABLE_ASSERTIONS)
 endif()
 
 if(FIZZY_FUZZING)
-    set(fuzzing_flags -fsanitize=fuzzer-no-link,address,undefined,nullability,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation)
+    if(NOT CMAKE_BUILD_TYPE STREQUAL Coverage)
+        set(fuzzing_flags -fsanitize=fuzzer-no-link,address,undefined,nullability,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation)
+    endif()
     add_compile_options(${fuzzing_flags})
     add_link_options(${fuzzing_flags})
 endif()

cmake/ProjectWabt.cmake

@@ -11,7 +11,7 @@ set(binary_dir ${prefix}/src/wabt-build)
 set(include_dir ${source_dir})
 set(wabt_library ${binary_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}wabt${CMAKE_STATIC_LIBRARY_SUFFIX})
 
-set(flags -fvisibility=hidden)
+set(flags "${fuzzing_flags} -fvisibility=hidden")
 if(SANITIZE MATCHES address)
     # Instrument WABT with ASan - required for container-overflow checks.
     set(flags "-fsanitize=address ${flags}")
@@ -24,12 +24,12 @@ endif()
 ExternalProject_Add(wabt
     EXCLUDE_FROM_ALL 1
     PREFIX ${prefix}
-    DOWNLOAD_NAME wabt-1.0.19.tar.gz
+    DOWNLOAD_NAME wabt-fixes.tar.gz
     DOWNLOAD_DIR ${prefix}/downloads
     SOURCE_DIR ${source_dir}
     BINARY_DIR ${binary_dir}
-    URL https://github.com/WebAssembly/wabt/archive/1.0.19.tar.gz
-    URL_HASH SHA256=134f2afc8205d0a3ab89c5f0d424ff3823e9d2769c39d2235aa37eba7abc15ba
+    URL https://github.com/ewasm/wabt/archive/fixes.tar.gz
+    URL_HASH SHA256=2df5d7aef95197e7c207935ff9318524b69f41458c1473a857f4771e8ec75e4c
     CMAKE_ARGS
     -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

cmake/spectests_contributing.md

@@ -0,0 +1,145 @@
+
+
+## Run fuzzer
+
+Run Fizzy vs WABT verification fuzzer, currently in `fuzzing` branch.
+Start with small inputs.
+
+```shell script
+bin/fizzy-fuzz-parser corpus -max_len=13
+```
+
+In case it finds a failure you will see output like
+```text
+INFO: Seed: 770491456
+INFO: Loaded 1 modules   (64068 inline 8-bit counters): 64068 [0xb8dc90, 0xb9d6d4), 
+INFO: Loaded 1 PC tables (64068 PCs): 64068 [0xb9d6d8,0xc97b18), 
+INFO:    10986 files found in corpus
+INFO: seed corpus: files: 10986 min: 1b max: 27419b total: 740862b rss: 40Mb
+#11160  INITED cov: 7782 ft: 9644 corp: 227/2677b exec/s: 0 rss: 186Mb
+#65536  pulse  cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 215Mb
+#131072 pulse  cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 248Mb
+#262144 pulse  cov: 7782 ft: 9644 corp: 227/2677b lim: 13 exec/s: 32768 rss: 314Mb
+  MALFORMED: invalid limits 4
+/home/chfast/Projects/wasmx/fizzy/test/fuzzer/parser_fuzzer.cpp:47:9: runtime error: execution reached an unreachable program point
+SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/chfast/Projects/wasmx/fizzy/test/fuzzer/parser_fuzzer.cpp:47:9 in 
+MS: 1 ChangeBit-; base unit: 846f4928f92649789961543e93544b4c2ed6fc44
+0x0,0x61,0x73,0x6d,0x1,0x0,0x0,0x0,0x5,0x3,0x1,0x4,0x2b,
+\x00asm\x01\x00\x00\x00\x05\x03\x01\x04+
+artifact_prefix='./'; Test unit written to ./crash-016934f781c41276bfacda9a90385cef85a88d5f
+Base64: AGFzbQEAAAAFAwEEKw==
+```
+
+In this example Fizzy reported the error `MALFORMED: invalid limits 4`
+while WABT considers the Wasm binary valid.
+The test unit (Wasm binary) is also printed in various forms and saved as
+`crash-016934f781c41276bfacda9a90385cef85a88d5f`.
+
+
+## Check WebAssembly spec interpreter
+
+In case of false negative validation result in WABT it is likely that
+such case is missing in the WAST spec test.
+
+Build the `wasm reference interpreter` following official instructions.
+
+Rename `crash-016934f781c41276bfacda9a90385cef85a88d5f` to `crash.wasm`
+as the interpreter need proper files extension to recognize file type.
+
+```shell script
+./wasm crash.wasm
+```
+
+Output:
+```text
+crash.wasm:0xb: decoding error: integer too large
+```
+
+Good news. This is effectively the same error reported as in Fizzy. In Wasm 1.0 `limits` kind
+can be 0 or 1. The the value 4 in the `crash.wasm` is larger than the max 1.
+
+
+## Inspect with WABT tools
+
+You can use `wasm2wat` or `wasm-validate` tools to confirm the bug in WABT.
+Make sure you use the same version as used for fuzzing.
+Disable all extensions except "mutable globals".
+
+In our example, `wasm2wat crash.wasm`:
+```webassembly
+(module
+  (memory (;0;) 43 i64))
+```
+This indicates that the issue is in the memory section.
+
+Unfortunately, we get different error as the data section also has invalid length.
+Let's fix that.
+
+
+## Prepare WAST test
+
+Dump the test unit (`xxd -p -c1000 crash.wasm`):
+
+```hexdump
+0061736d01000000050301042b
+```
+
+Here are created WAST tests, placed in `new.wast` file. In the original `crash.wasm` the invalid 4
+is followed by `2b` byte. We added 2 more variants of the test with byte `00` and without any 
+following byte (remember to change the section length if you actually modify the length).
+
+```wast
+(assert_malformed
+  (module binary
+    "\00asm" "\01\00\00\00"
+    "\05\03\01"                          ;; memory section
+    "\04\2b"                             ;; malformed memory limit 4
+  )
+  "integer too large"
+)
+
+(assert_malformed
+  (module binary
+    "\00asm" "\01\00\00\00"
+    "\05\03\01"                          ;; memory section
+    "\04\00"                             ;; malformed memory limit 4
+  )
+  "integer too large"
+)
+
+(assert_malformed
+  (module binary
+    "\00asm" "\01\00\00\00"
+    "\05\02\01"                          ;; memory section
+    "\04"                                ;; malformed memory limit 4
+  )
+  "integer too large"
+)
+```
+
+These tests can be rechecked with WABT.
+
+```shell script
+wast2json new.wast
+wasm-validate new.0.wasm
+wasm-validate new.1.wasm
+wasm-validate new.2.wasm
+```
+
+The last test case is actually failing in WABT for some other reason, but it can stay as a valid
+spectests addition.
+
+```text
+000000c: error: unable to read u32 leb128: memory initial page count
+```
+
+
+## Debug & fix WABT
+
+Debug `wasm-validate new.0.wasm` to find out why it passes validation.
+Binary loading code is in `binary-reader.cc`.
+
+The example bug is fixed by https://github.com/WebAssembly/wabt/pull/1547.
+
+
+## Send WAST tests upstream

test/CMakeLists.txt

@@ -22,6 +22,9 @@ if(HUNTER_ENABLED)
 endif()
 find_package(GTest REQUIRED)
 
+include(ProjectWabt)
+include(ProjectWasm3)
+
 add_subdirectory(utils)
 add_subdirectory(bench)
 add_subdirectory(bench_internal)

test/fuzzer/CMakeLists.txt

@@ -4,4 +4,4 @@
 
 add_executable(fizzy-fuzz-parser parser_fuzzer.cpp)
 target_link_options(fizzy-fuzz-parser PRIVATE -fsanitize=fuzzer)
-target_link_libraries(fizzy-fuzz-parser PRIVATE fizzy::fizzy-internal)
+target_link_libraries(fizzy-fuzz-parser PRIVATE fizzy::fizzy-internal wabt::wabt)

test/fuzzer/parser_fuzzer.cpp

@@ -3,18 +3,175 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "parser.hpp"
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
 
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t data_size) noexcept
+#include <src/binary-reader-ir.h>
+#include <src/binary-reader.h>
+#include <src/ir.h>
+#include <src/validator.h>
+
+namespace
+{
+struct Stats
+{
+    int64_t malformed = 0;
+    int64_t invalid = 0;
+    int64_t valid = 0;
+
+    ~Stats()
+    {
+        const auto all = malformed + invalid + valid;
+        if (all == 0)
+            return;
+        std::clog << "WASM STATS" << std::setprecision(3) << "\n  all:       " << all
+                  << "\n  malformed: " << malformed << " " << ((malformed * 100) / all) << "%"
+                  << "\n  invalid:   " << invalid << " " << ((invalid * 100) / all) << "%"
+                  << "\n  valid:     " << valid << " " << ((valid * 100) / all) << "%"
+                  << "\n";
+    }
+};
+
+Stats stats;
+
+void handle_unexpected_errors() noexcept
+{
+    static const bool ignore_errors = [] {
+        const auto options = std::getenv("OPTIONS");
+        if (!options)
+            return false;
+        return std::string{options}.find("ignore_errors") != std::string::npos;
+    }();
+    if (!ignore_errors)
+        __builtin_unreachable();
+}
+
+//constexpr auto wabt_ignored_errors = {
+//    "unable to read u32 leb128: version",
+//    "invalid linking metadata version:",
+//};
+
+wabt::Errors wabt_errors;
+
+bool wabt_parse(const uint8_t* data, size_t data_size) noexcept
+{
+    using namespace wabt;
+
+    ReadBinaryOptions read_options;
+    read_options.features.enable_mutable_globals();
+    read_options.features.disable_exceptions();
+    read_options.features.disable_annotations();
+    read_options.features.disable_bulk_memory();
+    read_options.features.disable_gc();
+    read_options.features.disable_memory64();
+    read_options.features.disable_multi_value();
+    read_options.features.disable_reference_types();
+    read_options.features.disable_sat_float_to_int();
+    read_options.features.disable_sign_extension();
+    read_options.features.disable_simd();
+    read_options.features.disable_tail_call();
+    read_options.features.disable_threads();
+    read_options.fail_on_custom_section_error = false;
+    read_options.stop_on_first_error = true;
+    read_options.read_debug_names = false;
+    Module module;
+
+    wabt_errors.clear();
+
+    {
+        const auto result =
+            ReadBinaryIr("fuzzing", data, data_size, read_options, &wabt_errors, &module);
+
+        if (Failed(result))
+            return false;
+
+        wabt_errors.clear();  // Clear errors (probably) from custom sections.
+    }
+
+    {
+        const auto result =
+            ValidateModule(&module, &wabt_errors, ValidateOptions{read_options.features});
+        if (Failed(result))
+            return false;
+    }
+
+    return wabt_errors.empty();
+}
+
+}  // namespace
+
+extern "C" {
+
+size_t LLVMFuzzerMutate(uint8_t* data, size_t size, size_t max_size) noexcept;
+
+size_t LLVMFuzzerCustomMutator(
+    uint8_t* data, size_t size, size_t max_size, [[maybe_unused]] unsigned int seed) noexcept
 {
+    static constexpr uint8_t wasm_prefix[]{0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00};
+    static constexpr auto wasm_prefix_size = sizeof(wasm_prefix);
+
+    // For inputs shorter than wasm prefix just mutate it.
+    if (size <= wasm_prefix_size)
+        return LLVMFuzzerMutate(data, size, max_size);
+
+    // For other, leave prefix unchanged. It is likely to be valid and we don't want to waste time
+    // on mutating the prefix.
+    const auto new_size_without_prefix = LLVMFuzzerMutate(
+        data + wasm_prefix_size, size - wasm_prefix_size, max_size - wasm_prefix_size);
+    return new_size_without_prefix + wasm_prefix_size;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t data_size) noexcept
+{
+    const auto expected = wabt_parse(data, data_size);
+
     try
     {
         fizzy::parse({data, data_size});
+        ++stats.valid;
+        if (!expected)
+        {
+            bool has_errors = false;
+            for (const auto& err : wabt_errors)
+            {
+//                bool ignored = false;
+//
+//                for (const auto& m : wabt_ignored_errors)
+//                {
+//                    if (err.message.find(m) != std::string::npos)
+//                        ignored = true;
+//                }
+//                if (ignored)
+//                    continue;
+
+                std::cerr << "  MISSED ERROR: " << err.message << "\n";
+                has_errors = true;
+            }
+
+            if (has_errors)
+                handle_unexpected_errors();
+        }
     }
-    catch (const fizzy::parser_error&)
+    catch (const fizzy::parser_error& err)
     {
+        ++stats.malformed;
+        if (expected)
+        {
+            std::cerr << "  MALFORMED: " << err.what() << "\n";
+            handle_unexpected_errors();
+        }
     }
-    catch (const fizzy::validation_error&)
+    catch (const fizzy::validation_error& err)
     {
+        ++stats.invalid;
+        if (expected)
+        {
+            std::cerr << "  INVALID: " << err.what() << "\n";
+            handle_unexpected_errors();
+        }
     }
+
     return 0;
 }
+}

test/utils/CMakeLists.txt

@@ -2,9 +2,6 @@
 # Copyright 2019-2020 The Fizzy Authors.
 # SPDX-License-Identifier: Apache-2.0
 
-include(ProjectWabt)
-include(ProjectWasm3)
-
 add_library(test-utils STATIC)
 add_library(fizzy::test-utils ALIAS test-utils)