Change test timeouts to be in realtime

src/PowerPlant.cpp

@@ -143,13 +143,13 @@ void PowerPlant::log(const LogLevel& level, std::stringstream& message) {
     log(level, message.str());
 }
 
-void PowerPlant::shutdown() {
+void PowerPlant::shutdown(bool force) {
 
     // Emit our shutdown event
     emit(std::make_unique<dsl::word::Shutdown>());
 
     // Shutdown the scheduler
-    scheduler.shutdown();
+    scheduler.shutdown(force);
 }
 
 }  // namespace NUClear

src/PowerPlant.hpp

@@ -118,8 +118,10 @@ class PowerPlant {
 
     /**
      * Shuts down the PowerPlant, tells all component threads to terminate and waits for them to finish.
+     *
+     * @param force If true, the PowerPlant will shutdown immediately without waiting for tasks to finish
      */
-    void shutdown();
+    void shutdown(bool force = false);
 
     /**
      * Installs a reactor of a particular type to the system.

src/threading/TaskScheduler.cpp

@@ -167,11 +167,14 @@ namespace threading {
         }
     }
 
-    void TaskScheduler::shutdown() {
+    void TaskScheduler::shutdown(bool force) {
         started.store(false, std::memory_order_release);
         running.store(false, std::memory_order_release);
         for (auto& pool : pool_queues) {
             const std::lock_guard<std::recursive_mutex> lock(pool.second->mutex);
+            if (force) {
+                pool.second->queue.clear();
+            }
             pool.second->condition.notify_all();
         }
     }

src/threading/TaskScheduler.hpp

@@ -164,9 +164,11 @@ namespace threading {
         /**
          *
          *  Shuts down the scheduler, all waiting threads are woken, and any attempt to get a task results in an
-         *  exception
+         *  exception.
+         *
+         * @param force If true, the scheduler will be shutdown immediately, and all tasks will be dropped
          */
-        void shutdown();
+        void shutdown(bool force = false);
 
         /**
          * Submit a new task to be executed to the Scheduler.

tests/test_util/TestBase.hpp

@@ -20,8 +20,8 @@
  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef TEST_UTIL_TESTBASE_HPP
-#define TEST_UTIL_TESTBASE_HPP
+#ifndef TEST_UTIL_TEST_BASE_HPP
+#define TEST_UTIL_TEST_BASE_HPP
 
 #include <catch2/catch_test_macros.hpp>
 #include <string>
@@ -32,7 +32,7 @@
 
 namespace test_util {
 
-template <typename BaseClass, int timeout = 1000>
+template <typename BaseClass>
 class TestBase : public NUClear::Reactor {
 public:
     /**
@@ -46,23 +46,58 @@ class TestBase : public NUClear::Reactor {
     template <int i>
     struct Step {};
 
-    explicit TestBase(std::unique_ptr<NUClear::Environment> environment, const bool& shutdown_on_idle = true)
+    /**
+     * Emit this struct to fail the test
+     */
+    struct Fail {
+        explicit Fail(std::string message) : message(std::move(message)) {}
+        std::string message;
+    };
+
+    explicit TestBase(std::unique_ptr<NUClear::Environment> environment,
+                      const bool& shutdown_on_idle             = true,
+                      const std::chrono::milliseconds& timeout = std::chrono::milliseconds(1000))
         : Reactor(std::move(environment)) {
 
         // Shutdown if the system is idle
         if (shutdown_on_idle) {
             on<Idle<>>().then([this] { powerplant.shutdown(); });
         }
 
+        on<Shutdown>().then([this] {
+            const std::lock_guard<std::mutex> lock(timeout_mutex);
+            clean_shutdown = true;
+            timeout_cv.notify_all();
+        });
+
         // Timeout if the test doesn't complete in time
-        on<Watchdog<BaseClass, timeout, std::chrono::milliseconds>, MainThread>().then([this] {
-            powerplant.shutdown();
-            INFO("Test timed out");
+        // Typically we would use a watchdog, however it is subject to Time Travel
+        // So instead spawn a thread that will wait for the timeout and then fail the test and shut down
+        on<Always>().then([this, timeout] {
+            if (clean_shutdown) {
+                return;
+            }
+            std::unique_lock<std::mutex> lock(timeout_mutex);
+            timeout_cv.wait_for(lock, timeout);
+
+            if (!clean_shutdown) {
+                powerplant.shutdown(true);
+                emit<Scope::DIRECT>(std::make_unique<Fail>("Test timed out"));
+            }
+        });
+
+        on<Trigger<Fail>, MainThread>().then([this](const Fail& f) {
+            INFO(f.message);
             CHECK(false);
         });
     }
+
+private:
+    std::mutex timeout_mutex;
+    std::condition_variable timeout_cv;
+    bool clean_shutdown = false;
 };
 
 }  // namespace test_util
 
-#endif  // TEST_UTIL_TESTBASE_HPP
+#endif  // TEST_UTIL_TEST_BASE_HPP

tests/tests/api/TimeTravel.cpp

@@ -23,7 +23,7 @@ struct Results {
     std::array<TimePair, 2> events;
 };
 
-class TestReactor : public test_util::TestBase<TestReactor, 5000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
 

tests/tests/api/TimeTravelFrozen.cpp

@@ -13,7 +13,7 @@ constexpr std::chrono::milliseconds SHUTDOWN_TIME = std::chrono::milliseconds(12
 
 struct WaitForShutdown {};
 
-class TestReactor : public test_util::TestBase<TestReactor, 5000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
 

tests/tests/dsl/Every.cpp

@@ -33,10 +33,11 @@ std::vector<NUClear::clock::time_point> every_times;    // NOLINT(cppcoreguideli
 std::vector<NUClear::clock::time_point> per_times;      // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 std::vector<NUClear::clock::time_point> dynamic_times;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
-class TestReactor : public test_util::TestBase<TestReactor, 20000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 
 public:
-    TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
+    TestReactor(std::unique_ptr<NUClear::Environment> environment)
+        : TestBase(std::move(environment), false, std::chrono::seconds(10)) {
 
         // Trigger on 3 different types of every
         on<Every<1000, Per<std::chrono::seconds>>>().then([]() { every_times.push_back(NUClear::clock::now()); });

tests/tests/dsl/Idle.cpp

@@ -37,7 +37,7 @@ struct SimpleMessage {
 
 constexpr int time_step = 50;
 
-class TestReactor : public test_util::TestBase<TestReactor, 10000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     template <int N>
     struct CustomPool {
@@ -51,7 +51,8 @@ class TestReactor : public test_util::TestBase<TestReactor, 10000> {
         emit(std::make_unique<Step<N + 1>>());
     }
 
-    TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
+    TestReactor(std::unique_ptr<NUClear::Environment> environment)
+        : TestBase(std::move(environment), false, std::chrono::seconds(2)) {
 
         start_time = NUClear::clock::now();
 

tests/tests/dsl/IdleSync.cpp

@@ -30,7 +30,7 @@ namespace {
 /// A vector of events that have happened
 std::vector<std::string> events;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
-class TestReactor : public test_util::TestBase<TestReactor, 1000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
 

tests/tests/dsl/TCP.cpp

@@ -57,7 +57,7 @@ struct TestConnection {
 
 struct Finished {};
 
-class TestReactor : public test_util::TestBase<TestReactor, 2000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     void handle_data(const std::string& name, const IO::Event& event) {
         // We have data to read
@@ -78,7 +78,8 @@ class TestReactor : public test_util::TestBase<TestReactor, 2000> {
         }
     }
 
-    TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
+    TestReactor(std::unique_ptr<NUClear::Environment> environment)
+        : TestBase(std::move(environment), false, std::chrono::seconds(2)) {
 
         for (const auto& t : active_tests) {
             switch (t) {

tests/tests/dsl/Watchdog.cpp

@@ -35,7 +35,7 @@ std::vector<std::string> events;  // NOLINT(cppcoreguidelines-avoid-non-const-gl
 template <int I>
 struct Flag {};
 
-class TestReactor : public test_util::TestBase<TestReactor, 10000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
     TestReactor(std::unique_ptr<NUClear::Environment> environment)
         : TestBase(std::move(environment), false), start(NUClear::clock::now()) {

tests/tests/dsl/emit/Delay.cpp

@@ -51,9 +51,10 @@ struct TargetTimeMessage {
 
 struct FinishTest {};
 
-class TestReactor : public test_util::TestBase<TestReactor, 2000> {
+class TestReactor : public test_util::TestBase<TestReactor> {
 public:
-    TestReactor(std::unique_ptr<NUClear::Environment> environment) : TestBase(std::move(environment), false) {
+    TestReactor(std::unique_ptr<NUClear::Environment> environment)
+        : TestBase(std::move(environment), false, std::chrono::seconds(2)) {
 
         // Measure when messages were sent and received and print those values
         on<Trigger<DelayedMessage>>().then([](const DelayedMessage& m) {