diff --git a/CMakeLists.txt b/CMakeLists.txt index 48ad9975..7c5b18af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ set(SNITCH_MAX_NESTED_SECTIONS 8 CACHE STRING "Maximum depth of nested sec set(SNITCH_MAX_EXPR_LENGTH 1024 CACHE STRING "Maximum length of a printed expression when reporting failure.") set(SNITCH_MAX_MESSAGE_LENGTH 1024 CACHE STRING "Maximum length of error or status messages.") set(SNITCH_MAX_TEST_NAME_LENGTH 1024 CACHE STRING "Maximum length of a test case name.") +set(SNITCH_MAX_TAG_LENGTH 256 CACHE STRING "Maximum length of a test tag.") set(SNITCH_MAX_CAPTURES 8 CACHE STRING "Maximum number of captured expressions in a test case.") set(SNITCH_MAX_CAPTURE_LENGTH 256 CACHE STRING "Maximum length of a captured expression.") set(SNITCH_MAX_UNIQUE_TAGS 1024 CACHE STRING "Maximum number of unique tags in a test application.") @@ -59,6 +60,7 @@ if (SNITCH_CREATE_LIBRARY) SNITCH_MAX_EXPR_LENGTH=${SNITCH_MAX_EXPR_LENGTH} SNITCH_MAX_MESSAGE_LENGTH=${SNITCH_MAX_MESSAGE_LENGTH} SNITCH_MAX_TEST_NAME_LENGTH=${SNITCH_MAX_TEST_NAME_LENGTH} + SNITCH_MAX_TAG_LENGTH=${SNITCH_MAX_TAG_LENGTH} SNITCH_MAX_UNIQUE_TAGS=${SNITCH_MAX_UNIQUE_TAGS} SNITCH_MAX_COMMAND_LINE_ARGS=${SNITCH_MAX_COMMAND_LINE_ARGS} SNITCH_DEFINE_MAIN=$ diff --git a/README.md b/README.md index daa95a2d..5bf0c98b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t - No heap allocation from the testing framework, so heap allocations from your code can be tracked precisely. - Works with exceptions disabled, albeit with a minor limitation (see [Exceptions](#exceptions) below). - No external dependency; just pure C++20 with the STL. - - Compiles template-heavy tests at least 60% faster than other testing frameworks (see [Benchmark](#benchmark)). + - Compiles template-heavy tests at least 50% faster than other testing frameworks (see Release [benchmarks](#benchmark)). - By defaults, test results are reported to the standard output, with optional coloring for readability. Test events can also be forwarded to a reporter callback for reporting to CI frameworks (Teamcity, ..., see [Reporters](#reporters)). - Limited subset of the [_Catch2_](https://github.com/catchorg/_Catch2_) API, see [Comparison with _Catch2_](#detailed-comparison-with-catch2). - Additional API not in _Catch2_, or different from _Catch2_: @@ -152,7 +152,7 @@ See the documentation for the [header-only mode](#header-only-build) for more in ## Benchmark The following benchmarks were done using real-world tests from another library ([_observable_unique_ptr_](https://github.com/cschreib/observable_unique_ptr)), which generates about 4000 test cases and 25000 checks. This library uses "typed" tests almost exclusively, where each test case is instantiated several times, each time with a different tested type (here, 25 types). Building and running the tests was done without parallelism to simplify the comparison. The benchmarks were ran on a desktop with the following specs: - - OS: Linux Mint 20.3, linux kernel 5.15.0-48-generic. + - OS: Linux Mint 20.3, linux kernel 5.15.0-56-generic. - CPU: AMD Ryzen 5 2600 (6 core). - RAM: 16GB. - Storage: NVMe. @@ -176,22 +176,22 @@ Results for Debug builds: | **Debug** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 1.7s | 64s | 2.0s | 0s | -| Build tests | 63s | 86s | 78s | 109s | -| Build all | 65s | 150s | 80s | 109s | -| Run tests | 18ms | 83ms | 60ms | 20ms | -| Library size | 2.90MB | 38.6MB | 2.8MB | 0MB | -| Executable size | 33.2MB | 49.3MB | 38.6MB | 51.9MB | +| Build framework | 2.0s | 41s | 2.0s | 0s | +| Build tests | 65s | 79s | 73s | 118s | +| Build all | 67s | 120s | 75s | 118s | +| Run tests | 31ms | 76ms | 63ms | 20ms | +| Library size | 3.3MB | 38.6MB | 2.8MB | 0MB | +| Executable size | 33.4MB | 49.3MB | 38.6MB | 51.9MB | Results for Release builds: | **Release** | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ | |-----------------|----------|----------|-----------|------------| -| Build framework | 2.4s | 68s | 3.6s | 0s | -| Build tests | 135s | 264s | 216s | 281s | -| Build all | 137s | 332s | 220s | 281s | -| Run tests | 9ms | 31ms | 36ms | 10ms | -| Library size | 0.62MB | 2.6MB | 0.39MB | 0MB | +| Build framework | 2.6s | 47s | 3.5s | 0s | +| Build tests | 137s | 254s | 207s | 289s | +| Build all | 140s | 301s | 210s | 289s | +| Run tests | 24ms | 46ms | 44ms | 5ms | +| Library size | 0.65MB | 2.6MB | 0.39MB | 0MB | | Executable size | 9.8MB | 17.4MB | 15.2MB | 11.3MB | Notes: @@ -216,7 +216,7 @@ This must be called at namespace, global, or class scope; not inside a function `TEMPLATE_TEST_CASE(NAME, TAGS, TYPES...) { /* test code for TestType */ }` -This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead. +This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. The full name of the test, used when filtering tests by name, is `"NAME "`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead. `TEMPLATE_LIST_TEST_CASE(NAME, TAGS, TYPES) { /* test code for TestType */ }` @@ -653,14 +653,54 @@ An example reporter for _Teamcity_ is included for demonstration, see `include/s ### Default main function The default `main()` function provided in _snitch_ offers the following command-line API: - - positional argument for filtering tests by name. + - positional arguments for filtering tests by name, see below. - `-h,--help`: show command line help. - `-l,--list-tests`: list all tests. - ` --list-tags`: list all tags. - ` --list-tests-with-tag`: list all tests with a given tag. - - `-t,--tags`: filter tests by tags instead of by name. - - `-v,--verbosity [quiet|normal|high]`: select level of detail for the default reporter. - - ` --color [always|never]`: enable/disable colors in the default reporter. + - `-v,--verbosity `: select level of detail for the default reporter. + - ` --color `: enable/disable colors in the default reporter. + +The positional arguments are used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If at least one filter is provided, then hidden tests will no longer be excluded by default. This reproduces the behavior of _Catch2_. + +A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example: + - `ab*` will include all test cases with names starting with `ab`. + - `*cd` will include all test cases with names ending with `cd`. + - `ab*cd` will include all test cases with names starting with `ab` and ending with `cd`. + - `abcd` will only include the test case with name `abcd`. + - `*` will include all test cases. + +If a filter starts with `~`, then it is interpreted as an exclusion: + - `~ab*` will exclude all test cases with names starting with `ab`. + - `~*cd` will exclude all test cases with names ending with `cd`. + - `~ab*cd` will exclude all test cases with names starting with `ab` and ending with `cd`. + - `~abcd` will exclude the test case with name `abcd`. + - `~*` will exclude all test cases. + +If a filter starts with `[` or `~[`, then it applies to the test case tags, else it applies to the test case name. This behavior can be bypassed by escaping the bracket `\[`, in which case the filter applies to the test case name again (see note below on escaping). + +Finally, if more than one filter is provided, then filters are applied one after the other, in the order provided. As in _Catch2_, a filter will include (or exclude with `~`) the tests that match the inclusion (or exclusion) pattern, but will leave the status of tests that do not match the filter unchanged. Filters on test names and tags can be mixed. For example, the table below shows which test is included (1) or excluded (0) after applying the three filters `a* ~*d abcd`: + +| Test name | Initial | Apply `a*` | State | Apply `~*d` | State | Apply `abcd` | State | +|-----------|---------| ------------|-------|-------------|-------|--------------|-------| +| `a` | 0 | 1 | 1 | | 1 | | 1 | +| `b` | 0 | | 0 | | 0 | | 0 | +| `c` | 0 | | 0 | | 0 | | 0 | +| `d` | 0 | | 0 | 0 | 0 | | 0 | +| `abc` | 0 | 1 | 1 | | 1 | | 1 | +| `abd` | 0 | 1 | 1 | 0 | 0 | | 0 | +| `abcd` | 0 | 1 | 1 | 0 | 0 | 1 | 1 | + +**Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash, like `\*`. In general, any character located after a single backslash will be interpreted as a regular character, with no special meaning. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_. The table below shows examples of how edge-cases are handled: + +| Bash | _snitch_ | matches | +|---------|----------|---------------------------------------------| +| `\\` | `\` | nothing (ill-formed filter) | +| `\\*` | `\*` | any name which is exactly the `*` character | +| `\\\\` | `\\` | any name which is exactly the `\` character | +| `\\\\*` | `\\*` | any name starting with the `\` character | +| `[a*` | `[a*` | any tag starting with `[a` | +| `\\[a*` | `\[a*` | any name starting with `[a` | ### Using your own main function diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp index 9406de5d..d337b6a0 100644 --- a/include/snitch/snitch.hpp +++ b/include/snitch/snitch.hpp @@ -33,6 +33,8 @@ constexpr std::size_t max_message_length = SNITCH_MAX_MESSAGE_LENGTH; // Maximum length of a full test case name. // The full test case name includes the base name, plus any type. constexpr std::size_t max_test_name_length = SNITCH_MAX_TEST_NAME_LENGTH; +// Maximum length of a tag, including brackets. +constexpr std::size_t max_tag_length = SNITCH_MAX_TAG_LENGTH; // Maximum number of captured expressions in a test case. constexpr std::size_t max_captures = SNITCH_MAX_CAPTURES; // Maximum length of a captured expression. @@ -580,6 +582,18 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept { [[nodiscard]] bool replace_all( small_string_span string, std::string_view pattern, std::string_view replacement) noexcept; +[[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept; + +enum class filter_result { included, excluded, not_included, not_excluded }; + +[[nodiscard]] filter_result +is_filter_match_name(std::string_view name, std::string_view filter) noexcept; + +[[nodiscard]] filter_result +is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept; + +[[nodiscard]] filter_result is_filter_match_id(const test_id& id, std::string_view filter) noexcept; + template concept matcher_for = requires(const T& m, const U& value) { { m.match(value) } -> convertible_to; @@ -1159,6 +1173,11 @@ std::optional get_option(const cli::input& args, std::string_view std::optional get_positional_argument(const cli::input& args, std::string_view name) noexcept; + +void for_each_positional_argument( + const cli::input& args, + std::string_view name, + const small_function& callback) noexcept; } // namespace snitch::cli // Test registry. @@ -1238,9 +1257,11 @@ class registry { impl::test_state run(impl::test_case& test) noexcept; - bool run_all_tests(std::string_view run_name) noexcept; - bool run_tests_matching_name(std::string_view run_name, std::string_view name_filter) noexcept; - bool run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept; + bool run_tests(std::string_view run_name) noexcept; + + bool run_selected_tests( + std::string_view run_name, + const small_function& filter) noexcept; bool run_tests(const cli::input& args) noexcept; diff --git a/include/snitch/snitch_config.hpp.config b/include/snitch/snitch_config.hpp.config index a69aea39..6f9dac9a 100644 --- a/include/snitch/snitch_config.hpp.config +++ b/include/snitch/snitch_config.hpp.config @@ -24,6 +24,9 @@ #if !defined(SNITCH_MAX_TEST_NAME_LENGTH) # define SNITCH_MAX_TEST_NAME_LENGTH ${SNITCH_MAX_TEST_NAME_LENGTH} #endif +#if !defined(SNITCH_MAX_TAG_LENGTH) +# define SNITCH_MAX_TAG_LENGTH ${SNITCH_MAX_TAG_LENGTH} +#endif #if !defined(SNITCH_MAX_CAPTURES) # define SNITCH_MAX_CAPTURES ${SNITCH_MAX_CAPTURES} #endif diff --git a/src/snitch.cpp b/src/snitch.cpp index cb5ef8f7..f8fa219f 100644 --- a/src/snitch.cpp +++ b/src/snitch.cpp @@ -218,6 +218,70 @@ bool replace_all( return !overflow; } } + +bool is_match(std::string_view string, std::string_view regex) noexcept { + // An empty regex matches any string; early exit. + // An empty string matches an empty regex (exit here) or any regex containing + // only wildcards (exit later). + if ((string.empty() && regex.empty()) || regex.empty()) { + return true; + } + + const std::size_t regex_size = regex.size(); + const std::size_t string_size = string.size(); + + // Iterate characters of the regex string and exit at first non-match. + std::size_t js = 0; + for (std::size_t jr = 0; jr < regex_size; ++jr, ++js) { + bool escaped = false; + if (regex[jr] == '\\') { + // Escaped character, look ahead ignoring special characters. + ++jr; + if (jr >= regex_size) { + // Nothing left to escape; the regex is ill-formed. + return false; + } + + escaped = true; + } + + if (!escaped && regex[jr] == '*') { + // Wildcard is found; if this is the last character of the regex + // then any further content will be a match; early exit. + if (jr == regex_size - 1) { + return true; + } + + // Discard what has already been matched. + regex = regex.substr(jr + 1); + + // If there are no more characters in the string after discarding, then we only match if + // the regex contains only wildcards from there on. + const std::size_t remaining = string_size >= js ? string_size - js : 0u; + if (remaining == 0u) { + return regex.find_first_not_of('*') == regex.npos; + } + + // Otherwise, we loop over all remaining characters of the string and look + // for a match when starting from each of them. + for (std::size_t o = 0; o < remaining; ++o) { + if (is_match(string.substr(js + o), regex)) { + return true; + } + } + + return false; + } else if (js >= string_size || regex[jr] != string[js]) { + // Regular character is found; not a match if not an exact match in the string. + return false; + } + } + + // We have finished reading the regex string and did not find either a definite non-match + // or a definite match. This means we did not have any wildcard left, hence that we need + // an exact match. Therefore, only match if the string size is the same as the regex. + return js == string_size; +} } // namespace snitch namespace snitch::impl { @@ -478,14 +542,6 @@ void for_each_raw_tag(std::string_view s, F&& callback) noexcept { callback(s.substr(last_pos)); } -std::string_view get_tag_name(std::string_view tag) { - if (tag.size() < 2u || tag[0] != '[' || tag[tag.size() - 1u] != ']') { - return {}; - } - - return tag.substr(1u, tag.size() - 2u); -} - namespace tags { struct ignored {}; struct may_fail {}; @@ -496,173 +552,66 @@ using parsed_tag = std::variant void for_each_tag(std::string_view s, F&& callback) noexcept { + small_string buffer; + for_each_raw_tag(s, [&](std::string_view t) { // Look for "ignore" tags, which is either "[.]" // or a a tag starting with ".", like "[.integration]". if (t == "[.]"sv) { // This is a pure "ignore" tag, add this to the list of special tags. callback(tags::parsed_tag{tags::ignored{}}); - return; + } else if (t.starts_with("[."sv)) { + // This is a combined "ignore" + normal tag, add the "ignore" to the list of special + // tags, and continue with the normal tag. + callback(tags::parsed_tag{tags::ignored{}}); + callback(tags::parsed_tag{std::string_view("[.]")}); + + buffer.clear(); + if (!append(buffer, "[", t.substr(2u))) { + terminate_with("tag is too long"); + } + + t = buffer; } if (t == "[!mayfail]") { callback(tags::parsed_tag{tags::may_fail{}}); - return; } if (t == "[!shouldfail]") { callback(tags::parsed_tag{tags::should_fail{}}); - return; - } - - if (t.starts_with("[."sv)) { - // This is a combined "ignore" + normal tag, add the "ignore" to the list of special - // tags, and continue with the normal tag. - callback(tags::parsed_tag{tags::ignored{}}); - t = t.substr(2u, t.size() - 3u); - } else { - t = t.substr(1u, t.size() - 2u); } callback(tags::parsed_tag(t)); }); } -template -bool run_tests(registry& r, std::string_view run_name, F&& predicate) noexcept { - if (!r.report_callback.empty()) { - r.report_callback(r, event::test_run_started{run_name}); - } else if (is_at_least(r.verbose, registry::verbosity::normal)) { - r.print( - make_colored("starting tests with ", r.with_color, color::highlight2), - make_colored("snitch v" SNITCH_FULL_VERSION "\n", r.with_color, color::highlight1)); - r.print("==========================================\n"); - } - - bool success = true; - std::size_t run_count = 0; - std::size_t fail_count = 0; - std::size_t skip_count = 0; - std::size_t assertion_count = 0; - -#if SNITCH_WITH_TIMINGS - using clock = std::chrono::high_resolution_clock; - auto time_start = clock::now(); -#endif - - for (test_case& t : r) { - if (!predicate(t)) { - continue; - } - - auto state = r.run(t); - - ++run_count; - assertion_count += state.asserts; - - switch (t.state) { - case impl::test_case_state::success: { - // Nothing to do - break; - } - case impl::test_case_state::failed: { - ++fail_count; - success = false; - break; - } - case impl::test_case_state::skipped: { - ++skip_count; - break; - } - case impl::test_case_state::not_run: { - // Unreachable - break; - } - } - } - -#if SNITCH_WITH_TIMINGS - auto time_end = clock::now(); - float duration = std::chrono::duration(time_end - time_start).count(); -#endif - - if (!r.report_callback.empty()) { -#if SNITCH_WITH_TIMINGS - r.report_callback( - r, event::test_run_ended{ - .name = run_name, - .success = success, - .run_count = run_count, - .fail_count = fail_count, - .skip_count = skip_count, - .assertion_count = assertion_count, - .duration = duration}); -#else - r.report_callback( - r, event::test_run_ended{ - .name = run_name, - .success = success, - .run_count = run_count, - .fail_count = fail_count, - .skip_count = skip_count, - .assertion_count = assertion_count}); -#endif - } else if (is_at_least(r.verbose, registry::verbosity::normal)) { - r.print("==========================================\n"); - - if (success) { - r.print( - make_colored("success:", r.with_color, color::pass), " all tests passed (", - run_count, " test cases, ", assertion_count, " assertions"); - } else { - r.print( - make_colored("error:", r.with_color, color::fail), " some tests failed (", - fail_count, " out of ", run_count, " test cases, ", assertion_count, " assertions"); +std::string_view +make_full_name(small_string& buffer, const test_id& id) noexcept { + buffer.clear(); + if (id.type.length() != 0) { + if (!append(buffer, id.name, " <", id.type, ">")) { + return {}; } - - if (skip_count > 0) { - r.print(", ", skip_count, " test cases skipped"); + } else { + if (!append(buffer, id.name)) { + return {}; } - -#if SNITCH_WITH_TIMINGS - r.print(", ", duration, " seconds"); -#endif - - r.print(")\n"); } - return success; + return buffer.str(); } template void list_tests(const registry& r, F&& predicate) noexcept { + small_string buffer; for (const test_case& t : r) { if (!predicate(t)) { continue; } - if (!t.id.type.empty()) { - r.print(t.id.name, " [", t.id.type, "]\n"); - } else { - r.print(t.id.name, "\n"); - } - } -} - -std::string_view -make_full_name(small_string& buffer, const test_id& id) noexcept { - buffer.clear(); - if (id.type.length() != 0) { - if (!append(buffer, id.name, " [", id.type, "]")) { - return {}; - } - } else { - if (!append(buffer, id.name)) { - return {}; - } + r.print(make_full_name(buffer, t.id), "\n"); } - - return buffer.str(); } void set_state(test_case& t, impl::test_case_state s) noexcept { @@ -692,6 +641,45 @@ small_vector make_capture_buffer(const capture_s } // namespace namespace snitch { +filter_result is_filter_match_name(std::string_view name, std::string_view filter) noexcept { + filter_result match_action = filter_result::included; + filter_result no_match_action = filter_result::not_included; + if (filter.starts_with('~')) { + filter = filter.substr(1); + match_action = filter_result::excluded; + no_match_action = filter_result::not_excluded; + } + + return is_match(name, filter) ? match_action : no_match_action; +} + +filter_result is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept { + filter_result match_action = filter_result::included; + filter_result no_match_action = filter_result::not_included; + if (filter.starts_with('~')) { + filter = filter.substr(1); + match_action = filter_result::excluded; + no_match_action = filter_result::not_excluded; + } + + bool match = false; + for_each_tag(tags, [&](const tags::parsed_tag& v) { + if (auto* vs = std::get_if(&v); vs != nullptr && is_match(*vs, filter)) { + match = true; + } + }); + + return match ? match_action : no_match_action; +} + +filter_result is_filter_match_id(const test_id& id, std::string_view filter) noexcept { + if (filter.starts_with('[') || filter.starts_with("~[")) { + return is_filter_match_tags(id.tags, filter); + } else { + return is_filter_match_name(id.name, filter); + } +} + const char* registry::add(const test_id& id, test_ptr func) noexcept { if (test_list.size() == test_list.capacity()) { print( @@ -915,7 +903,7 @@ test_state registry::run(test_case& test) noexcept { thread_current_test = &state; #if SNITCH_WITH_TIMINGS - using clock = std::chrono::high_resolution_clock; + using clock = std::chrono::steady_clock; auto time_start = clock::now(); #endif @@ -997,49 +985,127 @@ test_state registry::run(test_case& test) noexcept { return state; } -bool registry::run_all_tests(std::string_view run_name) noexcept { - return ::run_tests(*this, run_name, [](const test_case& t) { - bool selected = true; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& s) { - if (std::holds_alternative(s)) { - selected = false; - } - }); +bool registry::run_selected_tests( + std::string_view run_name, + const small_function& predicate) noexcept { - return selected; - }); -} + if (!report_callback.empty()) { + report_callback(*this, event::test_run_started{run_name}); + } else if (is_at_least(verbose, registry::verbosity::normal)) { + print( + make_colored("starting tests with ", with_color, color::highlight2), + make_colored("snitch v" SNITCH_FULL_VERSION "\n", with_color, color::highlight1)); + print("==========================================\n"); + } -bool registry::run_tests_matching_name( - std::string_view run_name, std::string_view name_filter) noexcept { - small_string buffer; - return ::run_tests(*this, run_name, [&](const test_case& t) { - std::string_view v = make_full_name(buffer, t.id); + bool success = true; + std::size_t run_count = 0; + std::size_t fail_count = 0; + std::size_t skip_count = 0; + std::size_t assertion_count = 0; - // TODO: use regex here? - return v.find(name_filter) != v.npos; - }); -} +#if SNITCH_WITH_TIMINGS + using clock = std::chrono::steady_clock; + auto time_start = clock::now(); +#endif -bool registry::run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept { - tag_filter = get_tag_name(tag_filter); - if (tag_filter.empty()) { - print( - make_colored("error:", with_color, color::fail), - " tag must be of the form '[tag_name]'."); - std::terminate(); + for (test_case& t : *this) { + if (!predicate(t.id)) { + continue; + } + + auto state = run(t); + + ++run_count; + assertion_count += state.asserts; + + switch (t.state) { + case impl::test_case_state::success: { + // Nothing to do + break; + } + case impl::test_case_state::failed: { + ++fail_count; + success = false; + break; + } + case impl::test_case_state::skipped: { + ++skip_count; + break; + } + case impl::test_case_state::not_run: { + // Unreachable + break; + } + } } - return ::run_tests(*this, run_name, [&](const test_case& t) { - bool selected = false; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); vs != nullptr && *vs == tag_filter) { - selected = true; +#if SNITCH_WITH_TIMINGS + auto time_end = clock::now(); + float duration = std::chrono::duration(time_end - time_start).count(); +#endif + + if (!report_callback.empty()) { +#if SNITCH_WITH_TIMINGS + report_callback( + *this, event::test_run_ended{ + .name = run_name, + .success = success, + .run_count = run_count, + .fail_count = fail_count, + .skip_count = skip_count, + .assertion_count = assertion_count, + .duration = duration}); +#else + report_callback( + *this, event::test_run_ended{ + .name = run_name, + .success = success, + .run_count = run_count, + .fail_count = fail_count, + .skip_count = skip_count, + .assertion_count = assertion_count}); +#endif + } else if (is_at_least(verbose, registry::verbosity::normal)) { + print("==========================================\n"); + + if (success) { + print( + make_colored("success:", with_color, color::pass), " all tests passed (", run_count, + " test cases, ", assertion_count, " assertions"); + } else { + print( + make_colored("error:", with_color, color::fail), " some tests failed (", fail_count, + " out of ", run_count, " test cases, ", assertion_count, " assertions"); + } + + if (skip_count > 0) { + print(", ", skip_count, " test cases skipped"); + } + +#if SNITCH_WITH_TIMINGS + print(", ", duration, " seconds"); +#endif + + print(")\n"); + } + + return success; +} + +bool registry::run_tests(std::string_view run_name) noexcept { + const auto filter = [](const test_id& id) { + bool selected = true; + for_each_tag(id.tags, [&](const tags::parsed_tag& s) { + if (std::holds_alternative(s)) { + selected = false; } }); return selected; - }); + }; + + return run_selected_tests(run_name, filter); } void registry::list_all_tags() const noexcept { @@ -1075,23 +1141,9 @@ void registry::list_all_tests() const noexcept { } void registry::list_tests_with_tag(std::string_view tag) const noexcept { - tag = get_tag_name(tag); - if (tag.empty()) { - print( - make_colored("error:", with_color, color::fail), - " tag must be of the form '[tag_name]'."); - std::terminate(); - } - list_tests(*this, [&](const test_case& t) { - bool selected = false; - for_each_tag(t.id.tags, [&](const tags::parsed_tag& v) { - if (auto* vs = std::get_if(&v); vs != nullptr && *vs == tag) { - selected = true; - } - }); - - return selected; + const auto result = is_filter_match_tags(t.id.tags, tag); + return result == filter_result::included || result == filter_result::not_excluded; }); } @@ -1126,13 +1178,15 @@ using namespace std::literals; constexpr std::size_t max_arg_names = 2; -enum class argument_type { optional, mandatory }; +namespace argument_type { +enum type { optional = 0b00, mandatory = 0b01, repeatable = 0b10 }; +} struct expected_argument { small_vector names; std::optional value_name; std::string_view description; - argument_type type = argument_type::optional; + argument_type::type type = argument_type::optional; }; using expected_arguments = small_vector; @@ -1152,6 +1206,26 @@ std::string_view extract_executable(std::string_view path) { return path; } +bool is_option(const expected_argument& e) { + return !e.names.empty(); +} + +bool is_option(const cli::argument& a) { + return !a.name.empty(); +} + +bool has_value(const expected_argument& e) { + return e.value_name.has_value(); +} + +bool is_mandatory(const expected_argument& e) { + return (e.type & argument_type::mandatory) != 0; +} + +bool is_repeatable(const expected_argument& e) { + return (e.type & argument_type::repeatable) != 0; +} + std::optional parse_arguments( int argc, const char* const argv[], @@ -1168,7 +1242,8 @@ std::optional parse_arguments( small_vector expected_found; for (const auto& e : expected) { expected_found.push_back(false); - if (!e.names.empty()) { + + if (is_option(e)) { if (e.names.size() == 1) { if (!e.names[0].starts_with('-')) { terminate_with("option name must start with '-' or '--'"); @@ -1179,7 +1254,7 @@ std::optional parse_arguments( } } } else { - if (!e.value_name.has_value()) { + if (!has_value(e)) { terminate_with("positional argument must have a value name"); } } @@ -1188,12 +1263,15 @@ std::optional parse_arguments( // Parse for (int argi = 1; argi < argc; ++argi) { std::string_view arg(argv[argi]); + if (arg.starts_with('-')) { + // Options start with dashes. bool found = false; + for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (e.names.empty()) { + if (!is_option(e)) { continue; } @@ -1203,7 +1281,7 @@ std::optional parse_arguments( found = true; - if (expected_found[arg_index]) { + if (expected_found[arg_index] && !is_repeatable(e)) { console_print( make_colored("error:", settings.with_color, color::error), " duplicate command line argument '", arg, "'\n"); @@ -1213,7 +1291,7 @@ std::optional parse_arguments( expected_found[arg_index] = true; - if (e.value_name) { + if (has_value(e)) { if (argi + 1 == argc) { console_print( make_colored("error:", settings.with_color, color::error), @@ -1239,11 +1317,17 @@ std::optional parse_arguments( " unknown command line argument '", arg, "'\n"); } } else { + // If no dash, this is a positional argument. bool found = false; + for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (!e.names.empty() || expected_found[arg_index]) { + if (is_option(e)) { + continue; + } + + if (expected_found[arg_index] && !is_repeatable(e)) { continue; } @@ -1265,8 +1349,8 @@ std::optional parse_arguments( for (std::size_t arg_index = 0; arg_index < expected.size(); ++arg_index) { const auto& e = expected[arg_index]; - if (e.type == argument_type::mandatory && !expected_found[arg_index]) { - if (e.names.empty()) { + if (!expected_found[arg_index] && is_mandatory(e)) { + if (!is_option(e)) { console_print( make_colored("error:", settings.with_color, color::error), " missing positional argument '<", *e.value_name, ">'\n"); @@ -1302,16 +1386,22 @@ void print_help( // Print command line usage example console_print(make_colored("Usage:", settings.with_color, color::pass), "\n"); console_print(" ", program_name); - if (std::any_of(expected.cbegin(), expected.cend(), [](auto& e) { return !e.names.empty(); })) { + if (std::any_of(expected.cbegin(), expected.cend(), [](auto& e) { return is_option(e); })) { console_print(" [options...]"); } for (const auto& e : expected) { - if (e.names.empty()) { - if (e.type == argument_type::mandatory) { + if (!is_option(e)) { + if (!is_mandatory(e) && !is_repeatable(e)) { + console_print(" [<", *e.value_name, ">]"); + } else if (is_mandatory(e) && !is_repeatable(e)) { console_print(" <", *e.value_name, ">"); + } else if (!is_mandatory(e) && is_repeatable(e)) { + console_print(" [<", *e.value_name, ">...]"); + } else if (is_mandatory(e) && is_repeatable(e)) { + console_print(" <", *e.value_name, ">..."); } else { - console_print(" [<", *e.value_name, ">]"); + terminate_with("unhandled argument type"); } } } @@ -1324,7 +1414,7 @@ void print_help( heading.clear(); bool success = true; - if (!e.names.empty()) { + if (is_option(e)) { if (e.names[0].starts_with("--")) { success = success && append(heading, " "); } @@ -1335,7 +1425,7 @@ void print_help( success = success && append(heading, ", ", e.names[1]); } - if (e.value_name) { + if (has_value(e)) { success = success && append(heading, " <", *e.value_name, ">"); } } else { @@ -1357,11 +1447,10 @@ constexpr expected_arguments expected_args = { {{"-l", "--list-tests"}, {}, "List tests by name"}, {{"--list-tags"}, {}, "List tags by name"}, {{"--list-tests-with-tag"}, {"[tag]"}, "List tests by name with a given tag"}, - {{"-t", "--tags"}, {}, "Use tags for filtering, not name"}, {{"-v", "--verbosity"}, {"quiet|normal|high"}, "Define how much gets sent to the standard output"}, {{"--color"}, {"always|never"}, "Enable/disable color in output"}, {{"-h", "--help"}, {}, "Print help"}, - {{}, {"test regex"}, "A regex to select which test cases (or tags) to run"}}; + {{}, {"test regex"}, "A regex to select which test cases to run", argument_type::repeatable}}; // clang-format on constexpr bool with_color_default = SNITCH_DEFAULT_WITH_COLOR == 1; @@ -1401,7 +1490,7 @@ get_positional_argument(const cli::input& args, std::string_view name) noexcept std::optional ret; auto iter = std::find_if(args.arguments.cbegin(), args.arguments.cend(), [&](const auto& arg) { - return arg.name.empty() && arg.value_name == name; + return !is_option(arg) && arg.value_name == name; }); if (iter != args.arguments.cend()) { @@ -1410,6 +1499,24 @@ get_positional_argument(const cli::input& args, std::string_view name) noexcept return ret; } + +void for_each_positional_argument( + const cli::input& args, + std::string_view name, + const small_function& callback) noexcept { + + auto iter = args.arguments.cbegin(); + while (iter != args.arguments.cend()) { + iter = std::find_if(iter, args.arguments.cend(), [&](const auto& arg) { + return !is_option(arg) && arg.value_name == name; + }); + + if (iter != args.arguments.cend()) { + callback(*iter->value); + ++iter; + } + } +} } // namespace snitch::cli namespace snitch { @@ -1465,14 +1572,35 @@ bool registry::run_tests(const cli::input& args) noexcept { return true; } - if (auto opt = get_positional_argument(args, "test regex")) { - if (get_option(args, "--tags")) { - return run_tests_with_tag(args.executable, *opt->value); - } else { - return run_tests_matching_name(args.executable, *opt->value); - } + if (get_positional_argument(args, "test regex").has_value()) { + const auto filter = [&](const test_id& id) noexcept { + std::optional selected; + + const auto callback = [&](std::string_view filter) noexcept { + switch (is_filter_match_id(id, filter)) { + case filter_result::included: selected = true; break; + case filter_result::excluded: selected = false; break; + case filter_result::not_included: + if (!selected.has_value()) { + selected = false; + } + break; + case filter_result::not_excluded: + if (!selected.has_value()) { + selected = true; + } + break; + } + }; + + for_each_positional_argument(args, "test regex", callback); + + return selected.value(); + }; + + return run_selected_tests(args.executable, filter); } else { - return run_all_tests(args.executable); + return run_tests(args.executable); } } } // namespace snitch diff --git a/tests/runtime_tests/cli.cpp b/tests/runtime_tests/cli.cpp index b2242011..db279166 100644 --- a/tests/runtime_tests/cli.cpp +++ b/tests/runtime_tests/cli.cpp @@ -156,14 +156,29 @@ TEST_CASE("parse arguments positional", "[cli]") { CHECK(console.messages.empty()); } -TEST_CASE("parse arguments too many positional", "[cli]") { +TEST_CASE("parse arguments multiple positional", "[cli]") { console_output_catcher console; const arg_vector args = {"test", "arg1", "arg2"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); - REQUIRE(!input.has_value()); - CHECK(console.messages == contains_substring("too many positional arguments")); + REQUIRE(input.has_value()); + CHECK(input->executable == "test"sv); + REQUIRE(input->arguments.size() == 2u); + + CHECK(input->arguments[0].name == ""sv); + REQUIRE(input->arguments[0].value.has_value()); + REQUIRE(input->arguments[0].value_name.has_value()); + CHECK(input->arguments[0].value.value() == "arg1"sv); + CHECK(input->arguments[0].value_name.value() == "test regex"sv); + + CHECK(input->arguments[1].name == ""sv); + REQUIRE(input->arguments[1].value.has_value()); + REQUIRE(input->arguments[1].value_name.has_value()); + CHECK(input->arguments[1].value.value() == "arg2"sv); + CHECK(input->arguments[1].value_name.value() == "test regex"sv); + + CHECK(console.messages.empty()); } TEST_CASE("get option", "[cli]") { @@ -200,6 +215,9 @@ TEST_CASE("get positional argument", "[cli]") { cli_input{"at middle"sv, {"test", "--help", "arg1", "--verbosity", "high"}}, cli_input{"at start"sv, {"test", "arg1", "--help", "--verbosity", "high"}}, cli_input{"alone"sv, {"test", "arg1"}}, + cli_input{"multiple"sv, {"test", "arg1", "arg2"}}, + cli_input{ + "multiple interleaved"sv, {"test", "arg1", "--verbosity", "high", "arg2"}}, }) { #if SNITCH_TEST_WITH_SNITCH @@ -214,6 +232,23 @@ TEST_CASE("get positional argument", "[cli]") { CHECK(arg->name == ""sv); CHECK(arg->value == "arg1"sv); CHECK(arg->value_name == "test regex"sv); + + if (input->executable.starts_with("multiple")) { + std::size_t i = 0u; + + auto callback = [&](std::string_view value) noexcept { + if (i == 0u) { + CHECK(value == "arg1"sv); + } else { + CHECK(value == "arg2"sv); + } + + ++i; + }; + + snitch::cli::for_each_positional_argument(*input, "test regex", callback); + CHECK(i == 2u); + } } } diff --git a/tests/runtime_tests/registry.cpp b/tests/runtime_tests/registry.cpp index e71e4c44..a72606f9 100644 --- a/tests/runtime_tests/registry.cpp +++ b/tests/runtime_tests/registry.cpp @@ -146,8 +146,8 @@ TEST_CASE("add template test", "[registry]") { CHECK(test_called == false); CHECK(test_called_int == true); CHECK(test_called_float == false); - CHECK(framework.messages == contains_substring("starting: how many lights [int]")); - CHECK(framework.messages == contains_substring("finished: how many lights [int]")); + CHECK(framework.messages == contains_substring("starting: how many lights ")); + CHECK(framework.messages == contains_substring("finished: how many lights ")); } SECTION("run float default reporter") { @@ -157,8 +157,8 @@ TEST_CASE("add template test", "[registry]") { CHECK(test_called == false); CHECK(test_called_int == false); CHECK(test_called_float == true); - CHECK(framework.messages == contains_substring("starting: how many lights [float]")); - CHECK(framework.messages == contains_substring("finished: how many lights [float]")); + CHECK(framework.messages == contains_substring("starting: how many lights ")); + CHECK(framework.messages == contains_substring("finished: how many lights ")); } SECTION("run int custom reporter") { @@ -503,8 +503,8 @@ TEST_CASE("run tests", "[registry]") { INFO((r == reporter::print ? "default reporter" : "custom reporter")); - SECTION("run all tests") { - framework.registry.run_all_tests("test_app"); + SECTION("run tests") { + framework.registry.run_tests("test_app"); CHECK(test_called); CHECK(test_called_other_tag); @@ -526,7 +526,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all pass") { - framework.registry.run_tests_matching_name("test_app", "are you"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*are you") == + snitch::filter_result::included; + }); CHECK(test_called); CHECK(!test_called_other_tag); @@ -547,7 +551,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all failed") { - framework.registry.run_tests_matching_name("test_app", "lights"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*lights*") == + snitch::filter_result::included; + }); CHECK(!test_called); CHECK(test_called_other_tag); @@ -568,7 +576,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered all skipped") { - framework.registry.run_tests_matching_name("test_app", "cup"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_name(id.name, "*cup") == + snitch::filter_result::included; + }); CHECK(!test_called); CHECK(!test_called_other_tag); @@ -590,7 +602,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests filtered tags") { - framework.registry.run_tests_with_tag("test_app", "[other_tag]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[other_tag]") == + snitch::filter_result::included; + }); CHECK(!test_called); CHECK(test_called_other_tag); @@ -610,8 +626,37 @@ TEST_CASE("run tests", "[registry]") { } } + SECTION("run tests filtered tags wildcard") { + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "*tag]") == + snitch::filter_result::included; + }); + + CHECK(test_called); + CHECK(test_called_other_tag); + CHECK(test_called_skipped); + CHECK(test_called_int); + CHECK(test_called_float); + CHECK(test_called_hidden1); + CHECK(!test_called_hidden2); + + if (r == reporter::print) { + CHECK( + framework.messages == + contains_substring("some tests failed (3 out of 6 test cases, 3 assertions")); + } else { + CHECK(framework.get_num_runs() == 6u); + CHECK_RUN(false, 6u, 3u, 1u, 3u); + } + } + SECTION("run tests special tag [.]") { - framework.registry.run_tests_with_tag("test_app", "[hidden]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[hidden]") == + snitch::filter_result::included; + }); CHECK(!test_called); CHECK(!test_called_other_tag); @@ -632,7 +677,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!mayfail]") { - framework.registry.run_tests_with_tag("test_app", "[may fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[may fail]") == + snitch::filter_result::included; + }); if (r == reporter::print) { CHECK( @@ -645,7 +694,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!shouldfail]") { - framework.registry.run_tests_with_tag("test_app", "[should fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[should fail]") == + snitch::filter_result::included; + }); if (r == reporter::print) { CHECK( @@ -658,7 +711,11 @@ TEST_CASE("run tests", "[registry]") { } SECTION("run tests special tag [!shouldfail][!mayfail]") { - framework.registry.run_tests_with_tag("test_app", "[may+should fail]"); + framework.registry.run_selected_tests( + "test_app", [](const snitch::test_id& id) noexcept { + return snitch::is_filter_match_tags(id.tags, "[may+should fail]") == + snitch::filter_result::included; + }); if (r == reporter::print) { CHECK( @@ -683,8 +740,8 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); - CHECK(framework.messages == contains_substring("how many templated lights [float]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK(framework.messages == contains_substring("hidden test 1")); CHECK(framework.messages == contains_substring("hidden test 2")); } @@ -697,14 +754,16 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("[other_tag]")); CHECK(framework.messages == contains_substring("[tag with spaces]")); CHECK(framework.messages == contains_substring("[hidden]")); - CHECK(framework.messages != contains_substring("[.]")); + CHECK(framework.messages == contains_substring("[.]")); CHECK(framework.messages != contains_substring("[.hidden]")); + CHECK(framework.messages == contains_substring("[!shouldfail]")); + CHECK(framework.messages == contains_substring("[!mayfail]")); } SECTION("list_tests_with_tag") { for (auto tag : {"[tag]"sv, "[other_tag]"sv, "[skipped]"sv, "[tag with spaces]"sv, "[wrong_tag]"sv, - "[hidden]"sv, "[.]"sv, "[.hidden]"sv}) { + "[hidden]"sv, "[.]"sv, "[.hidden]"sv, "*tag]"sv}) { CAPTURE(tag); framework.messages.clear(); @@ -714,34 +773,40 @@ TEST_CASE("list tests", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK( - framework.messages == contains_substring("how many templated lights [float]")); + framework.messages == contains_substring("how many templated lights ")); } else if (tag == "[other_tag]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); CHECK( - framework.messages != contains_substring("how many templated lights [float]")); + framework.messages != contains_substring("how many templated lights ")); } else if (tag == "[skipped]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages != contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); CHECK( - framework.messages != contains_substring("how many templated lights [float]")); + framework.messages != contains_substring("how many templated lights ")); } else if (tag == "[tag with spaces]"sv) { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages != contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); CHECK( - framework.messages == contains_substring("how many templated lights [float]")); - } else if (tag == "[hidden]"sv) { + framework.messages == contains_substring("how many templated lights ")); + } else if (tag == "[hidden]"sv || tag == "[.]"sv) { CHECK(framework.messages == contains_substring("hidden test 1")); CHECK(framework.messages == contains_substring("hidden test 2")); - } else if (tag == "[wrong_tag]"sv || tag == "[.]"sv || tag == "[.hidden]"sv) { + } else if (tag == "*tag]"sv) { + CHECK(framework.messages == contains_substring("how are you")); + CHECK(framework.messages == contains_substring("how many lights")); + CHECK(framework.messages == contains_substring("drink from the cup")); + CHECK(framework.messages == contains_substring("how many templated lights")); + CHECK(framework.messages == contains_substring("hidden test 1")); + } else if (tag == "[wrong_tag]"sv || tag == "[.hidden]"sv) { CHECK(framework.messages.empty()); } } @@ -842,8 +907,8 @@ TEST_CASE("run tests cli", "[registry]") { CHECK(framework.messages == contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages == contains_substring("drink from the cup")); - CHECK(framework.messages == contains_substring("how many templated lights [int]")); - CHECK(framework.messages == contains_substring("how many templated lights [float]")); + CHECK(framework.messages == contains_substring("how many templated lights ")); + CHECK(framework.messages == contains_substring("how many templated lights ")); } SECTION("--list-tags") { @@ -867,23 +932,56 @@ TEST_CASE("run tests cli", "[registry]") { CHECK(framework.messages != contains_substring("how are you")); CHECK(framework.messages == contains_substring("how many lights")); CHECK(framework.messages != contains_substring("drink from the cup")); - CHECK(framework.messages != contains_substring("how many templated lights [int]")); - CHECK(framework.messages != contains_substring("how many templated lights [float]")); + CHECK(framework.messages != contains_substring("how many templated lights ")); + CHECK(framework.messages != contains_substring("how many templated lights ")); } SECTION("test filter") { - const arg_vector args = {"test", "how many"}; + const arg_vector args = {"test", "how many*"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); framework.registry.run_tests(*input); CHECK_RUN(false, 3u, 3u, 0u, 3u); } + SECTION("test filter exclusion") { + const arg_vector args = {"test", "~*fail"}; + auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); + framework.registry.run_tests(*input); + + CHECK_RUN(false, 7u, 3u, 1u, 3u); + } + SECTION("test tag filter") { - const arg_vector args = {"test", "--tags", "[skipped]"}; + const arg_vector args = {"test", "[skipped]"}; auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); framework.registry.run_tests(*input); CHECK_RUN(true, 1u, 0u, 1u, 0u); } } + +std::array readme_test_called = {false}; + +TEST_CASE("run tests cli readme example", "[registry]") { + mock_framework framework; + framework.setup_reporter_and_print(); + console_output_catcher console; + + readme_test_called = {false}; + + framework.registry.add({"a"}, []() { readme_test_called[0] = true; }); + framework.registry.add({"b"}, []() { readme_test_called[1] = true; }); + framework.registry.add({"c"}, []() { readme_test_called[2] = true; }); + framework.registry.add({"d"}, []() { readme_test_called[3] = true; }); + framework.registry.add({"abc"}, []() { readme_test_called[4] = true; }); + framework.registry.add({"abd"}, []() { readme_test_called[5] = true; }); + framework.registry.add({"abcd"}, []() { readme_test_called[6] = true; }); + + const arg_vector args = {"test", "a*", "~*d", "abcd"}; + auto input = snitch::cli::parse_arguments(static_cast(args.size()), args.data()); + framework.registry.run_tests(*input); + + std::array expected = {true, false, false, false, true, false, true}; + CHECK(readme_test_called == expected); +} diff --git a/tests/runtime_tests/string_utility.cpp b/tests/runtime_tests/string_utility.cpp index 9827d0b6..d79901aa 100644 --- a/tests/runtime_tests/string_utility.cpp +++ b/tests/runtime_tests/string_utility.cpp @@ -355,3 +355,232 @@ TEMPLATE_TEST_CASE( CHECK(std::string_view(s) == "abaca"); } } + +TEST_CASE("is_match", "[utility]") { + SECTION("empty") { + CHECK(snitch::is_match(""sv, ""sv)); + } + + SECTION("empty regex") { + CHECK(snitch::is_match("abc"sv, ""sv)); + } + + SECTION("empty string") { + CHECK(!snitch::is_match(""sv, "abc"sv)); + } + + SECTION("no wildcard match") { + CHECK(snitch::is_match("abc"sv, "abc"sv)); + } + + SECTION("no wildcard not match") { + CHECK(!snitch::is_match("abc"sv, "cba"sv)); + } + + SECTION("no wildcard not match smaller regex") { + CHECK(!snitch::is_match("abc"sv, "ab"sv)); + CHECK(!snitch::is_match("abc"sv, "bc"sv)); + CHECK(!snitch::is_match("abc"sv, "a"sv)); + CHECK(!snitch::is_match("abc"sv, "b"sv)); + CHECK(!snitch::is_match("abc"sv, "c"sv)); + } + + SECTION("no wildcard not match larger regex") { + CHECK(!snitch::is_match("abc"sv, "abcd"sv)); + CHECK(!snitch::is_match("abc"sv, "zabc"sv)); + CHECK(!snitch::is_match("abc"sv, "abcdefghijkl"sv)); + } + + SECTION("single wildcard match") { + CHECK(snitch::is_match("abc"sv, "*"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "*"sv)); + CHECK(snitch::is_match(""sv, "*"sv)); + } + + SECTION("start wildcard match") { + CHECK(snitch::is_match("abc"sv, "*bc"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "*bc"sv)); + CHECK(snitch::is_match("bc"sv, "*bc"sv)); + } + + SECTION("start wildcard not match") { + CHECK(!snitch::is_match("abd"sv, "*bc"sv)); + CHECK(!snitch::is_match("azzzzzzzzzzbd"sv, "*bc"sv)); + CHECK(!snitch::is_match("bd"sv, "*bc"sv)); + } + + SECTION("end wildcard match") { + CHECK(snitch::is_match("abc"sv, "ab*"sv)); + CHECK(snitch::is_match("abccccccccccc"sv, "ab*"sv)); + CHECK(snitch::is_match("ab"sv, "ab*"sv)); + } + + SECTION("end wildcard not match") { + CHECK(!snitch::is_match("adc"sv, "ab*"sv)); + CHECK(!snitch::is_match("adccccccccccc"sv, "ab*"sv)); + CHECK(!snitch::is_match("ad"sv, "ab*"sv)); + } + + SECTION("mid wildcard match") { + CHECK(snitch::is_match("ab_cd"sv, "ab*cd"sv)); + CHECK(snitch::is_match("abasdasdasdcd"sv, "ab*cd"sv)); + CHECK(snitch::is_match("abcd"sv, "ab*cd"sv)); + } + + SECTION("mid wildcard not match") { + CHECK(!snitch::is_match("adcd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abcc"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("accccccccccd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("ab"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abc"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("abd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("cd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("bcd"sv, "ab*cd"sv)); + CHECK(!snitch::is_match("acd"sv, "ab*cd"sv)); + } + + SECTION("multi wildcard match") { + CHECK(snitch::is_match("zab_cdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zzzzzzabcccccccccccdwwwwwww"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("ab_cdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zabcdw"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("zab_cd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("ababcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcdabcd"sv, "*ab*cd*"sv)); + CHECK(snitch::is_match("abcdabcc"sv, "*ab*cd*"sv)); + } + + SECTION("multi wildcard not match") { + CHECK(!snitch::is_match("zad_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zac_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zaa_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zdb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zcb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zbb_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_ddw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_bdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_adw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_ccw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_cbw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_caw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab_"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("zab"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("ab_"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("ab"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("_cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("cdw"sv, "*ab*cd*"sv)); + CHECK(!snitch::is_match("cd"sv, "*ab*cd*"sv)); + } + + SECTION("double wildcard match") { + CHECK(snitch::is_match("abc"sv, "**"sv)); + CHECK(snitch::is_match("azzzzzzzzzzbc"sv, "**"sv)); + CHECK(snitch::is_match(""sv, "**"sv)); + CHECK(snitch::is_match("abcdefg"sv, "*g*******"sv)); + CHECK(snitch::is_match("abc"sv, "abc**"sv)); + CHECK(snitch::is_match("abc"sv, "ab**"sv)); + CHECK(snitch::is_match("abc"sv, "a**"sv)); + CHECK(snitch::is_match("abc"sv, "**abc"sv)); + CHECK(snitch::is_match("abc"sv, "**bc"sv)); + CHECK(snitch::is_match("abc"sv, "**c"sv)); + CHECK(snitch::is_match("abc"sv, "ab**c"sv)); + CHECK(snitch::is_match("abc"sv, "a**bc"sv)); + CHECK(snitch::is_match("abc"sv, "a**c"sv)); + } + + SECTION("double wildcard not match") { + CHECK(!snitch::is_match("abc"sv, "abd**"sv)); + CHECK(!snitch::is_match("abc"sv, "ad**"sv)); + CHECK(!snitch::is_match("abc"sv, "d**"sv)); + CHECK(!snitch::is_match("abc"sv, "**abd"sv)); + CHECK(!snitch::is_match("abc"sv, "**bd"sv)); + CHECK(!snitch::is_match("abc"sv, "**d"sv)); + CHECK(!snitch::is_match("abc"sv, "ab**d"sv)); + CHECK(!snitch::is_match("abc"sv, "a**d"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**abc"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**ab"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**a"sv)); + CHECK(!snitch::is_match("abc"sv, "abc**def"sv)); + } + + SECTION("string contains wildcard & escaped wildcard") { + CHECK(snitch::is_match("a*c"sv, "a\\*c"sv)); + CHECK(snitch::is_match("a*"sv, "a\\*"sv)); + CHECK(snitch::is_match("*a"sv, "\\*a"sv)); + CHECK(snitch::is_match("a*"sv, "a*"sv)); + CHECK(snitch::is_match("a\\b"sv, "a\\\\b"sv)); + CHECK(snitch::is_match("a"sv, "\\a"sv)); + CHECK(!snitch::is_match("a"sv, "a\\"sv)); + CHECK(!snitch::is_match("a"sv, "a\\\\"sv)); + CHECK(!snitch::is_match("a"sv, "\\\\a"sv)); + } +} + +using snitch::filter_result; +using snitch::is_filter_match_id; +using snitch::is_filter_match_name; +using snitch::is_filter_match_tags; + +TEST_CASE("is_filter_match", "[utility]") { + CHECK(is_filter_match_name("abc"sv, "abc"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "ab*"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "*bc"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "*"sv) == filter_result::included); + CHECK(is_filter_match_name("abc"sv, "def"sv) == filter_result::not_included); + CHECK(is_filter_match_name("abc"sv, "~abc"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~ab*"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~*bc"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~*"sv) == filter_result::excluded); + CHECK(is_filter_match_name("abc"sv, "~def"sv) == filter_result::not_excluded); +} + +TEST_CASE("is_filter_match_tag", "[utility]") { + CHECK(is_filter_match_tags("[tag1]"sv, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag2]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag*]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tug*]"sv) == filter_result::not_excluded); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][.tag2]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[.tag1][tag2]"sv, "[.]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[.]"sv) == filter_result::not_excluded); + CHECK(is_filter_match_tags("[tag1][!mayfail]"sv, "[!mayfail]"sv) == filter_result::included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[!mayfail]"sv) == filter_result::not_excluded); + CHECK( + is_filter_match_tags("[tag1][!shouldfail]"sv, "[!shouldfail]"sv) == + filter_result::included); + CHECK( + is_filter_match_tags("[tag1][tag2]"sv, "~[!shouldfail]"sv) == filter_result::not_excluded); + + CHECK(is_filter_match_tags("[tag1]"sv, "[tag2]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tag3]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[tug*]*"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "[.]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[.tag1][tag2]"sv, "[.tag1]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag1]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2][.]"sv, "[.tag2]"sv) == filter_result::not_included); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tag1]"sv) == filter_result::excluded); + CHECK(is_filter_match_tags("[tag1][tag2]"sv, "~[tag2]"sv) == filter_result::excluded); +} + +TEST_CASE("is_filter_match_id", "[utility]") { + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "abc"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~abc"sv) == filter_result::excluded); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "ab*"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag1]"sv) == filter_result::included); + CHECK(is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag2]"sv) == filter_result::included); + CHECK( + is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "[tag3]"sv) == filter_result::not_included); + CHECK( + is_filter_match_id({"abc"sv, "[tag1][tag2]"sv}, "~[tag3]"sv) == + filter_result::not_excluded); + CHECK( + is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "\\[weird]"sv) == + filter_result::included); + CHECK( + is_filter_match_id({"[weird]"sv, "[tag1][tag2]"sv}, "[weird]"sv) == + filter_result::not_included); +} diff --git a/tests/testing_event.hpp b/tests/testing_event.hpp index 7f8b5079..1156afc5 100644 --- a/tests/testing_event.hpp +++ b/tests/testing_event.hpp @@ -49,7 +49,7 @@ struct mock_framework { .func = nullptr, .state = snitch::impl::test_case_state::not_run}; - snitch::small_vector events; + snitch::small_vector events; snitch::small_string<4086> messages; void report(const snitch::registry&, const snitch::event::data& e) noexcept;