Implemented time-limited command and test execution (fixes #157) (#183)

* implemented time limit in commands

* bug fix: moved time_limit to simple test suite

* added time limits to test execution

* version bump

bugzoo/mgr/container.py

@@ -283,9 +283,14 @@ def execute(self,
         Returns:
             the outcome of the test execution.
         """
-        bug = self.__installation.bugs[container.bug]
+        bug = self.__installation.bugs[container.bug] # type: Bug
         cmd, context = bug.harness.command(test)
-        response = self.command(container, cmd, context, stderr=True, verbose=verbose)
+        response = self.command(container,
+                                cmd=cmd,
+                                context=context,
+                                stderr=True,
+                                time_limit=bug.harness.time_limit, # TODO migrate
+                                verbose=verbose)
         passed = response.code == 0
         return TestOutcome(response, passed)
 
@@ -385,8 +390,12 @@ def command(self,
         if context is None:
             context = os.path.join(bug.source_dir, '..')
 
-        template_cmd = '/bin/bash -c "source /.environment && cd {} && {}"'
-        cmd = template_cmd.format(context, cmd)
+        cmd = 'source /.environment && cd {} && {}'.format(context, cmd)
+        cmd_wrapped = "/bin/bash -c '{}'".format(cmd)
+        if time_limit is not None and time_limit > 0:
+            logger.debug("running command with time limit: {} seconds".format(time_limit))
+            cmd_wrapped = "timeout {} {}".format(time_limit, cmd_wrapped)
+        cmd = cmd_wrapped
 
         # based on: https://github.com/roidelapluie/docker-py/commit/ead9ffa34193281967de8cc0d6e1c0dcbf50eda5
         logger.debug("executing command: %s", cmd)
@@ -404,12 +413,6 @@ def command(self,
         if not block:
             return PendingExecResponse(response, out)
 
-        # while self.__api_docker.exec_inspect(exec_id)['Running']:
-        #    time_running = timer() - time_start
-        #    if time_limit and time_running > time_limit:
-        #        # TODO kill exec
-        #        raise TimeoutError()
-
         output = []
         for line in out:
             line = line.decode('utf-8').rstrip('\n')

bugzoo/testing/simple.py

@@ -33,7 +33,7 @@ def from_yaml(yml: dict) -> 'SimpleTestSuite':
     def __init__(self,
                  command: str,
                  context: str,
-                 time_limit: float,
+                 time_limit: int,
                  test_names: List[str]
                  ) -> None:
         assert test_names != []
@@ -60,6 +60,13 @@ def command(self, test : TestCase) -> Tuple[str, str]:
         cmd = self.__command.replace('__ID__', test.name, 1)
         return (cmd, self.__context)
 
+    @property
+    def time_limit(self) -> int:
+        """
+        The time limit on individual test case executions, measured in seconds.
+        """
+        return self.__time_limit
+
 
 class GenProgTestSuite(SimpleTestSuite):
     @staticmethod
@@ -111,22 +118,15 @@ def failing(self) -> Iterator[TestCase]:
         return self.__failing.__iter__()
 
     @property
-    def num_passing(self):
+    def num_passing(self) -> int:
         """
         The number of passing tests in the test suite.
         """
         return len(self.__passing)
 
     @property
-    def num_failing(self):
+    def num_failing(self) -> int:
         """
         The number of failing tests in the test suite.
         """
         return len(self.__failing)
-
-    @property
-    def time_limit(self):
-        """
-        The time limit on individual test case executions, measured in seconds.
-        """
-        return self.__time_limit

bugzoo/version.py

@@ -1 +1 @@
-__version__ = '2.0.10'
+__version__ = '2.0.11'