Merge pull request #56 from cschreib/update

Add support for wildcards in CLI test and tag filters
snitch-org · Jan 16, 2023 · ea200a0 · ea200a0
2 parents c1bb1bd + 3497bc6
commit ea200a0
Show file tree

Hide file tree

Showing 9 changed files with 832 additions and 276 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -7,6 +7,7 @@ set(SNITCH_MAX_NESTED_SECTIONS    8    CACHE STRING "Maximum depth of nested sec
 set(SNITCH_MAX_EXPR_LENGTH        1024 CACHE STRING "Maximum length of a printed expression when reporting failure.")
 set(SNITCH_MAX_MESSAGE_LENGTH     1024 CACHE STRING "Maximum length of error or status messages.")
 set(SNITCH_MAX_TEST_NAME_LENGTH   1024 CACHE STRING "Maximum length of a test case name.")
+set(SNITCH_MAX_TAG_LENGTH         256  CACHE STRING "Maximum length of a test tag.")
 set(SNITCH_MAX_CAPTURES           8    CACHE STRING "Maximum number of captured expressions in a test case.")
 set(SNITCH_MAX_CAPTURE_LENGTH     256  CACHE STRING "Maximum length of a captured expression.")
 set(SNITCH_MAX_UNIQUE_TAGS        1024 CACHE STRING "Maximum number of unique tags in a test application.")
@@ -59,6 +60,7 @@ if (SNITCH_CREATE_LIBRARY)
     SNITCH_MAX_EXPR_LENGTH=${SNITCH_MAX_EXPR_LENGTH}
     SNITCH_MAX_MESSAGE_LENGTH=${SNITCH_MAX_MESSAGE_LENGTH}
     SNITCH_MAX_TEST_NAME_LENGTH=${SNITCH_MAX_TEST_NAME_LENGTH}
+    SNITCH_MAX_TAG_LENGTH=${SNITCH_MAX_TAG_LENGTH}
     SNITCH_MAX_UNIQUE_TAGS=${SNITCH_MAX_UNIQUE_TAGS}
     SNITCH_MAX_COMMAND_LINE_ARGS=${SNITCH_MAX_COMMAND_LINE_ARGS}
     SNITCH_DEFINE_MAIN=$<BOOL:${SNITCH_DEFINE_MAIN}>

diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ The goal of _snitch_ is to be a simple, cheap, non-invasive, and user-friendly t
  - No heap allocation from the testing framework, so heap allocations from your code can be tracked precisely.
  - Works with exceptions disabled, albeit with a minor limitation (see [Exceptions](#exceptions) below).
  - No external dependency; just pure C++20 with the STL.
- - Compiles template-heavy tests at least 60% faster than other testing frameworks (see [Benchmark](#benchmark)).
+ - Compiles template-heavy tests at least 50% faster than other testing frameworks (see Release [benchmarks](#benchmark)).
  - By defaults, test results are reported to the standard output, with optional coloring for readability. Test events can also be forwarded to a reporter callback for reporting to CI frameworks (Teamcity, ..., see [Reporters](#reporters)).
  - Limited subset of the [_Catch2_](https://github.com/catchorg/_Catch2_) API, see [Comparison with _Catch2_](#detailed-comparison-with-catch2).
  - Additional API not in _Catch2_, or different from _Catch2_:
@@ -152,7 +152,7 @@ See the documentation for the [header-only mode](#header-only-build) for more in
 ## Benchmark
 
 The following benchmarks were done using real-world tests from another library ([_observable_unique_ptr_](https://github.com/cschreib/observable_unique_ptr)), which generates about 4000 test cases and 25000 checks. This library uses "typed" tests almost exclusively, where each test case is instantiated several times, each time with a different tested type (here, 25 types). Building and running the tests was done without parallelism to simplify the comparison. The benchmarks were ran on a desktop with the following specs:
- - OS: Linux Mint 20.3, linux kernel 5.15.0-48-generic.
+ - OS: Linux Mint 20.3, linux kernel 5.15.0-56-generic.
  - CPU: AMD Ryzen 5 2600 (6 core).
  - RAM: 16GB.
  - Storage: NVMe.
@@ -176,22 +176,22 @@ Results for Debug builds:
 
 | **Debug**       | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ |
 |-----------------|----------|----------|-----------|------------|
-| Build framework | 1.7s     | 64s      | 2.0s      | 0s         |
-| Build tests     | 63s      | 86s      | 78s       | 109s       |
-| Build all       | 65s      | 150s     | 80s       | 109s       |
-| Run tests       | 18ms     | 83ms     | 60ms      | 20ms       |
-| Library size    | 2.90MB   | 38.6MB   | 2.8MB     | 0MB        |
-| Executable size | 33.2MB   | 49.3MB   | 38.6MB    | 51.9MB     |
+| Build framework | 2.0s     | 41s      | 2.0s      | 0s         |
+| Build tests     | 65s      | 79s      | 73s       | 118s       |
+| Build all       | 67s      | 120s     | 75s       | 118s       |
+| Run tests       | 31ms     | 76ms     | 63ms      | 20ms       |
+| Library size    | 3.3MB    | 38.6MB   | 2.8MB     | 0MB        |
+| Executable size | 33.4MB   | 49.3MB   | 38.6MB    | 51.9MB     |
 
 Results for Release builds:
 
 | **Release**     | _snitch_ | _Catch2_ | _doctest_ | _Boost UT_ |
 |-----------------|----------|----------|-----------|------------|
-| Build framework | 2.4s     | 68s      | 3.6s      | 0s         |
-| Build tests     | 135s     | 264s     | 216s      | 281s       |
-| Build all       | 137s     | 332s     | 220s      | 281s       |
-| Run tests       | 9ms      | 31ms     | 36ms      | 10ms       |
-| Library size    | 0.62MB   | 2.6MB    | 0.39MB    | 0MB        |
+| Build framework | 2.6s     | 47s      | 3.5s      | 0s         |
+| Build tests     | 137s     | 254s     | 207s      | 289s       |
+| Build all       | 140s     | 301s     | 210s      | 289s       |
+| Run tests       | 24ms     | 46ms     | 44ms      | 5ms        |
+| Library size    | 0.65MB   | 2.6MB    | 0.39MB    | 0MB        |
 | Executable size | 9.8MB    | 17.4MB   | 15.2MB    | 11.3MB     |
 
 Notes:
@@ -216,7 +216,7 @@ This must be called at namespace, global, or class scope; not inside a function
 
 `TEMPLATE_TEST_CASE(NAME, TAGS, TYPES...) { /* test code for TestType */ }`
 
-This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead.
+This is similar to `TEST_CASE`, except that it declares a new test case for each of the types listed in `TYPES...`. Within the test body, the current type can be accessed as `TestType`. The full name of the test, used when filtering tests by name, is `"NAME <TYPE>"`. If you tend to reuse the same list of types for multiple test cases, then `TEMPLATE_LIST_TEST_CASE()` is recommended instead.
 
 
 `TEMPLATE_LIST_TEST_CASE(NAME, TAGS, TYPES) { /* test code for TestType */ }`
@@ -653,14 +653,54 @@ An example reporter for _Teamcity_ is included for demonstration, see `include/s
 ### Default main function
 
 The default `main()` function provided in _snitch_ offers the following command-line API:
- - positional argument for filtering tests by name.
+ - positional arguments for filtering tests by name, see below.
  - `-h,--help`: show command line help.
  - `-l,--list-tests`: list all tests.
  - `   --list-tags`: list all tags.
  - `   --list-tests-with-tag`: list all tests with a given tag.
- - `-t,--tags`: filter tests by tags instead of by name.
- - `-v,--verbosity [quiet|normal|high]`: select level of detail for the default reporter.
- - `   --color [always|never]`: enable/disable colors in the default reporter.
+ - `-v,--verbosity <quiet|normal|high>`: select level of detail for the default reporter.
+ - `   --color <always|never>`: enable/disable colors in the default reporter.
+
+The positional arguments are used to select which tests to run. If no positional argument is given, all tests will be run, except those that are explicitly hidden with special tags (see [Tags](#tags)). If at least one filter is provided, then hidden tests will no longer be excluded by default. This reproduces the behavior of _Catch2_.
+
+A filter may contain any number of "wildcard" character, `*`, which can represent zero or more characters. For example:
+ - `ab*` will include all test cases with names starting with `ab`.
+ - `*cd` will include all test cases with names ending with `cd`.
+ - `ab*cd` will include all test cases with names starting with `ab` and ending with `cd`.
+ - `abcd` will only include the test case with name `abcd`.
+ - `*` will include all test cases.
+
+If a filter starts with `~`, then it is interpreted as an exclusion:
+ - `~ab*` will exclude all test cases with names starting with `ab`.
+ - `~*cd` will exclude all test cases with names ending with `cd`.
+ - `~ab*cd` will exclude all test cases with names starting with `ab` and ending with `cd`.
+ - `~abcd` will exclude the test case with name `abcd`.
+ - `~*` will exclude all test cases.
+
+If a filter starts with `[` or `~[`, then it applies to the test case tags, else it applies to the test case name. This behavior can be bypassed by escaping the bracket `\[`, in which case the filter applies to the test case name again (see note below on escaping).
+
+Finally, if more than one filter is provided, then filters are applied one after the other, in the order provided. As in _Catch2_, a filter will include (or exclude with `~`) the tests that match the inclusion (or exclusion) pattern, but will leave the status of tests that do not match the filter unchanged. Filters on test names and tags can be mixed. For example, the table below shows which test is included (1) or excluded (0) after applying the three filters `a* ~*d abcd`:
+
+| Test name | Initial |  Apply `a*` | State | Apply `~*d` | State | Apply `abcd` | State |
+|-----------|---------| ------------|-------|-------------|-------|--------------|-------|
+| `a`       | 0       |  1          | 1     |             | 1     |              | 1     |
+| `b`       | 0       |             | 0     |             | 0     |              | 0     |
+| `c`       | 0       |             | 0     |             | 0     |              | 0     |
+| `d`       | 0       |             | 0     | 0           | 0     |              | 0     |
+| `abc`     | 0       |  1          | 1     |             | 1     |              | 1     |
+| `abd`     | 0       |  1          | 1     | 0           | 0     |              | 0     |
+| `abcd`    | 0       |  1          | 1     | 0           | 0     | 1            | 1     |
+
+**Note:** To match the actual character `*` in a test name, the `*` in the filter must be escaped using a backslash, like `\*`. In general, any character located after a single backslash will be interpreted as a regular character, with no special meaning. Be mindful that most shells (Bash, etc.) will also require the backslash itself be escaped to be interpreted as an actual backslash in _snitch_. The table below shows examples of how edge-cases are handled:
+
+| Bash    | _snitch_ | matches                                     |
+|---------|----------|---------------------------------------------|
+| `\\`    | `\`      | nothing (ill-formed filter)                 |
+| `\\*`   | `\*`     | any name which is exactly the `*` character |
+| `\\\\`  | `\\`     | any name which is exactly the `\` character |
+| `\\\\*` | `\\*`    | any name starting with the `\` character    |
+| `[a*`   | `[a*`    | any tag starting with `[a`                  |
+| `\\[a*` | `\[a*`   | any name starting with `[a`                 |
 
 
 ### Using your own main function

diff --git a/include/snitch/snitch.hpp b/include/snitch/snitch.hpp
@@ -33,6 +33,8 @@ constexpr std::size_t max_message_length = SNITCH_MAX_MESSAGE_LENGTH;
 // Maximum length of a full test case name.
 // The full test case name includes the base name, plus any type.
 constexpr std::size_t max_test_name_length = SNITCH_MAX_TEST_NAME_LENGTH;
+// Maximum length of a tag, including brackets.
+constexpr std::size_t max_tag_length = SNITCH_MAX_TAG_LENGTH;
 // Maximum number of captured expressions in a test case.
 constexpr std::size_t max_captures = SNITCH_MAX_CAPTURES;
 // Maximum length of a captured expression.
@@ -580,6 +582,18 @@ bool append_or_truncate(small_string_span ss, Args&&... args) noexcept {
 [[nodiscard]] bool replace_all(
     small_string_span string, std::string_view pattern, std::string_view replacement) noexcept;
 
+[[nodiscard]] bool is_match(std::string_view string, std::string_view regex) noexcept;
+
+enum class filter_result { included, excluded, not_included, not_excluded };
+
+[[nodiscard]] filter_result
+is_filter_match_name(std::string_view name, std::string_view filter) noexcept;
+
+[[nodiscard]] filter_result
+is_filter_match_tags(std::string_view tags, std::string_view filter) noexcept;
+
+[[nodiscard]] filter_result is_filter_match_id(const test_id& id, std::string_view filter) noexcept;
+
 template<typename T, typename U>
 concept matcher_for = requires(const T& m, const U& value) {
                           { m.match(value) } -> convertible_to<bool>;
@@ -1159,6 +1173,11 @@ std::optional<cli::argument> get_option(const cli::input& args, std::string_view
 
 std::optional<cli::argument>
 get_positional_argument(const cli::input& args, std::string_view name) noexcept;
+
+void for_each_positional_argument(
+    const cli::input&                                      args,
+    std::string_view                                       name,
+    const small_function<void(std::string_view) noexcept>& callback) noexcept;
 } // namespace snitch::cli
 
 // Test registry.
@@ -1238,9 +1257,11 @@ class registry {
 
     impl::test_state run(impl::test_case& test) noexcept;
 
-    bool run_all_tests(std::string_view run_name) noexcept;
-    bool run_tests_matching_name(std::string_view run_name, std::string_view name_filter) noexcept;
-    bool run_tests_with_tag(std::string_view run_name, std::string_view tag_filter) noexcept;
+    bool run_tests(std::string_view run_name) noexcept;
+
+    bool run_selected_tests(
+        std::string_view                                     run_name,
+        const small_function<bool(const test_id&) noexcept>& filter) noexcept;
 
     bool run_tests(const cli::input& args) noexcept;
 

diff --git a/include/snitch/snitch_config.hpp.config b/include/snitch/snitch_config.hpp.config
@@ -24,6 +24,9 @@
 #if !defined(SNITCH_MAX_TEST_NAME_LENGTH)
 #    define SNITCH_MAX_TEST_NAME_LENGTH ${SNITCH_MAX_TEST_NAME_LENGTH}
 #endif
+#if !defined(SNITCH_MAX_TAG_LENGTH)
+#    define SNITCH_MAX_TAG_LENGTH ${SNITCH_MAX_TAG_LENGTH}
+#endif
 #if !defined(SNITCH_MAX_CAPTURES)
 #    define SNITCH_MAX_CAPTURES ${SNITCH_MAX_CAPTURES}
 #endif