From fd8cededf55049e4cf4e335e4d70d7f1c1b115f1 Mon Sep 17 00:00:00 2001 From: stelios Date: Sat, 18 Jun 2022 17:29:35 +0300 Subject: [PATCH] Modifications to aglais_benchmark for logging error messages --- aglais_benchmark/aglais_benchmark.py | 36 +++++++++++----------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/aglais_benchmark/aglais_benchmark.py b/aglais_benchmark/aglais_benchmark.py index a6c1cd3..ac294e8 100644 --- a/aglais_benchmark/aglais_benchmark.py +++ b/aglais_benchmark/aglais_benchmark.py @@ -94,6 +94,7 @@ def run_notebook(self, filepath, name, concurrent=False, delete=True): output = [] notebookid = None result = "" + messages = [] try: @@ -121,8 +122,8 @@ def run_notebook(self, filepath, name, concurrent=False, delete=True): text = result[0] notebookid = text.split(": ")[1] except Exception as e: - print ("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) - print (zdairi_result.decode()) + messages.append("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) + messages.append(zdairi_result.decode()) # Temporary fix.. If notebook failed to create, try once more if not notebookid: @@ -135,8 +136,8 @@ def run_notebook(self, filepath, name, concurrent=False, delete=True): text = result[0] notebookid = text.split(": ")[1] except Exception as e: - print ("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) - print (zdairi_result.decode()) + messages.append("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) + messages.append(zdairi_result.decode()) @@ -164,8 +165,8 @@ def run_notebook(self, filepath, name, concurrent=False, delete=True): break except Exception as e: status = "FAIL" - print ("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) - print (zdairi_output.decode()) + messages.append("Exception encountered while trying to create a notebook: " + tmpfile + " for user in config: " + config) + messages.append(zdairi_output.decode()) if status == "FAILED": status = "FAIL" @@ -174,7 +175,7 @@ def run_notebook(self, filepath, name, concurrent=False, delete=True): end = time.time() endtime_iso = datetime.now() - return (status, msg, end-start, output, starttime_iso.strftime('%Y-%m-%dT%H:%M:%S.%f%z'), endtime_iso.strftime('%Y-%m-%dT%H:%M:%S.%f%z'), notebookid, config) + return (status, msg, end-start, output, starttime_iso.strftime('%Y-%m-%dT%H:%M:%S.%f%z'), endtime_iso.strftime('%Y-%m-%dT%H:%M:%S.%f%z'), notebookid, config, messages) def run(self, concurrent=False, users=1, delay_start=0, delay_notebook=0, delete=True): @@ -196,19 +197,9 @@ def run(self, concurrent=False, users=1, delay_start=0, delay_notebook=0, delete if self.verbose: print ("Test started [Single User]") - results = [self._run_single(0, False, delay_start, delay_notebook, delete)] + results = self._run_single(0, False, delay_start, delay_notebook, delete) end = time.time() - result = "PASS" - for res in results: - if not res: - result = "FAIL" - break - - for k,v in res.items(): - if v["result"] != "PASS": - result = v["result"] - break if self.verbose: print ("Test completed! ({:.2f} seconds)".format(end-start)) @@ -241,7 +232,7 @@ def _run_single(self, iterable=0, concurrent=False, delay_start=0, delay_noteboo :rtype: dict """ - results = {} + results = [] time.sleep(delay_start * iterable ) created_notebooks = [] totaltime = 0 @@ -257,11 +248,12 @@ def _run_single(self, iterable=0, concurrent=False, delay_start=0, delay_noteboo msg = "" percent_change = 0 start, finish = (0,0) + messages = [] try: generated_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - result, msg, totaltime, output, start, finish, notebookid, user_config = self.run_notebook(filepath, generated_name, concurrent, delete) + result, msg, totaltime, output, start, finish, notebookid, user_config, messages = self.run_notebook(filepath, generated_name, concurrent, delete) created_notebooks.append([notebookid, user_config]) if totaltime > expectedtime: @@ -277,13 +269,13 @@ def _run_single(self, iterable=0, concurrent=False, delay_start=0, delay_noteboo msg += "Expected/Actual output missmatch of cell #" + str(i) + "! " percent_change = "{:.2f}".format(((totaltime - expectedtime)/expectedtime)*100) - results[name] = {"result" : result, "outputs" : {"valid" : output_valid}, "time" : {"result" : timing_status, "elapsed" : "{:.2f}".format(totaltime), "expected" : "{:.2f}".format(expectedtime), "percent" : percent_change, "start" : start, "finish": finish }, "logs" : msg} + results.append({"name" : name, "result" : result, "outputs" : {"valid" : output_valid}, "messages" : messages, "time" : {"result" : timing_status, "elapsed" : "{:.2f}".format(totaltime), "expected" : "{:.2f}".format(expectedtime), "percent" : percent_change, "start" : start, "finish": finish }, "logs" : msg}) except Exception as e: logging.exception(e) result = "FAIL" output_valid = False - results[name] = {"result" : result, "outputs" : {"valid" : output_valid}, "time" : {"result" : timing_status, "elapsed" : "{:.2f}".format(totaltime), "expected" : "{:.2f}".format(expectedtime), "percent" : percent_change, "start" : start, "finish": finish }, "logs" : msg } + results.append({"name" : name, "result" : result, "outputs" : {"valid" : output_valid}, "messages" : messages, "time" : {"result" : timing_status, "elapsed" : "{:.2f}".format(totaltime), "expected" : "{:.2f}".format(expectedtime), "percent" : percent_change, "start" : start, "finish": finish }, "logs" : msg }) time.sleep(delay_notebook)