diff --git a/.github/scripts/build-default-bitstreams.py b/.github/scripts/build-default-bitstreams.py index 56a468e0e..155d02551 100755 --- a/.github/scripts/build-default-bitstreams.py +++ b/.github/scripts/build-default-bitstreams.py @@ -4,7 +4,7 @@ from fabric.api import prefix, run, settings, execute # type: ignore import fabric_cfg -from ci_variables import ci_env +from ci_variables import ci_env, remote_fsim_dir, remote_cy_dir from github_common import upload_binary_file from typing import List, Tuple diff --git a/.github/scripts/run-linux-poweroff.py b/.github/scripts/run-linux-poweroff.py index 3ca6be59c..88289fb8e 100755 --- a/.github/scripts/run-linux-poweroff.py +++ b/.github/scripts/run-linux-poweroff.py @@ -3,7 +3,7 @@ from fabric.api import prefix, run, settings, execute # type: ignore import fabric_cfg -from ci_variables import ci_env +from ci_variables import ci_env, remote_fsim_dir, remote_cy_dir from utils import search_match_in_last_workloads_output_file, create_args, FpgaPlatform args = create_args() @@ -30,45 +30,34 @@ def run_w_timeout(workload_path, config_runtime, workload, timeout, num_passes): print(f"Starting workload run {workload}.") log_tail_length = 300 - def run_line(cmd): - rc = 0 - with settings(warn_only=True): - # pty=False needed to avoid issues with screen -ls stalling in fabric - rc = run(cmd, pty=False).return_code - return rc - - def print_last_n_log(): - print(f"Printing last {log_tail_length} lines of log.") - run(f"""cd {remote_fsim_dir}/deploy/log && LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""") - def run_firesim_cmd(typ, extra_args): + timeout_prefix = f"timeout {timeout} " firesim_opts = f"-c {workload_path}/{config_runtime} -a {remote_cy_dir}/sims/firesim-staging/sample_config_hwdb.yaml -r {remote_cy_dir}/sims/firesim-staging/sample_config_build_recipes.yaml" - return run_line(f"{timeout_prefix} firesim {firesim_opts} {extra_args} {typ}", pty=False) + return run_warn_only(f"{timeout_prefix} firesim {firesim_opts} {extra_args} {typ}", pty=False) rc = run_firesim_cmd("launchrunfarm") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) rc = run_firesim_cmd("infrasetup") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) rc = run_firesim_cmd("runworkload") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) # This is a janky solution to the fact the manager does not # return a non-zero exit code or some sort of result summary. # The expectation here is that the PR author will manually # check these output files for correctness until it can be # done programmatically.. - print(f"Printing last {log_tail_length} lines of all output files. See results-workload for more info.") - run(f"""cd {remote_fsim_dir}/deploy/results-workload/ && LAST_DIR=$(ls | tail -n1) && if [ -d "$LAST_DIR" ]; then tail -n{log_tail_length} $LAST_DIR/*/*; fi""") + print_last_firesim_workload(log_tail_length) # need to confirm that instance is off print("Terminating runfarm. Assuming this will pass.") run_firesim_cmd("terminaterunfarm", "-q") - print_last_n_log() + print_last_firesim_log(log_tail_length) # using rc of runworkload if rc != 0: diff --git a/.github/scripts/run-metasims.py b/.github/scripts/run-metasims.py index cd41a0ad2..4534158c2 100755 --- a/.github/scripts/run-metasims.py +++ b/.github/scripts/run-metasims.py @@ -4,7 +4,7 @@ from fabric.api import prefix, run, settings, execute # type: ignore import fabric_cfg -from ci_variables import ci_env +from ci_variables import ci_env, remote_fsim_dir, remote_cy_dir def run_parallel_metasim(): """ Runs parallel baremetal metasimulations """ @@ -29,51 +29,38 @@ def run_w_timeout(workload: str, timeout: str): :arg: timeout (str) - timeout amount for the workload to run """ log_tail_length = 300 - # unique tag based on the ci workflow and filename is needed to ensure - # run farm is unique to each linux-poweroff test - script_name = Path(__file__).stem - timeout_prefix = f"timeout {timeout} " - - def run_line(cmd): - rc = 0 - with settings(warn_only=True): - # pty=False needed to avoid issues with screen -ls stalling in fabric - rc = run(cmd, pty=False).return_code - return rc - - def print_last_n_log(): - print(f"Printing last {log_tail_length} lines of log.") - run(f"""cd {remote_fsim_dir}/deploy/log && LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""") def run_firesim_cmd(typ, extra_args): + timeout_prefix = f"timeout {timeout} " firesim_opts = f"-c {workload} -a {remote_cy_dir}/sims/firesim-staging/sample_config_hwdb.yaml -r {remote_cy_dir}/sims/firesim-staging/sample_config_build_recipes.yaml" - return run_line(f"{timeout_prefix} firesim {firesim_opts} {extra_args} {typ}", pty=False) + return run_warn_only(f"{timeout_prefix} firesim {firesim_opts} {extra_args} {typ}", pty=False) - with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_env['GITHUB_RUN_ID']}-{script_name}"): + # unique tag based on the ci workflow and filename is needed to ensure + # run farm is unique to each linux-poweroff test + with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_env['GITHUB_RUN_ID']}-{Path(__file__).stem}"): rc = run_firesim_cmd("launchrunfarm") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) rc = run_firesim_cmd("infrasetup") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) rc = run_firesim_cmd("runworkload") if rc != 0: - print_last_n_log() + print_last_firesim_log(log_tail_length) # This is a janky solution to the fact the manager does not # return a non-zero exit code or some sort of result summary. # The expectation here is that the PR author will manually # check these output files for correctness until it can be # done programmatically.. - print(f"Printing last {log_tail_length} lines of all output files. See results-workload for more info.") - run(f"""cd {remote_fsim_dir}/deploy/results-workload/ && LAST_DIR=$(ls | tail -n1) && if [ -d "$LAST_DIR" ]; then tail -n{log_tail_length} $LAST_DIR/*/*; fi""") + print_last_firesim_workload(log_tail_length) # need to confirm that instance is off print("Terminating runfarm. Assuming this will pass.") run_firesim_cmd("terminaterunfarm", "-q") - print_last_n_log() + print_last_firesim_log(log_tail_length) # using rc of runworkload if rc != 0: diff --git a/.github/scripts/utils.py b/.github/scripts/utils.py index 47b3f1c58..cae9ca8d1 100644 --- a/.github/scripts/utils.py +++ b/.github/scripts/utils.py @@ -2,6 +2,7 @@ from enum import Enum import argparse import os +from ci_variables import remote_firesim_dir def search_match_in_last_workloads_output_file(file_name: str = "uartlog", match_key: str = "*** PASSED ***") -> int: # if grep doesn't find any results, this command will fail @@ -37,3 +38,19 @@ def setup_shell_env_vars(): export_shell_env_vars.add(f"{v}={os.environ[v]}") return ("export " + " ".join(export_shell_env_vars)) if export_shell_env_vars else "true" + +def run_warn_only(cmd): + rc = 0 + with settings(warn_only=True): + # pty=False needed to avoid issues with screen -ls stalling in fabric + rc = run(cmd, pty=False).return_code + return rc + +def print_last_firesim_log(log_lines = 300): + print(f"Printing last {log_lines} lines of log.") + run(f"""cd {remote_fsim_dir}/deploy/log && LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""") + +def print_last_firesim_workload(log_lines = 300): + workload_path = f"{remote_firesim_dir}/deploy/results-workload" + print(f"Printing last {log_lines} lines of all output files. See {workload_path} for more info.") + run(f"""cd {workload_path} && LAST_DIR=$(ls | tail -n1) && if [ -d "$LAST_DIR" ]; then tail -n{log_lines} $LAST_DIR/*/*; fi""")