diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index a9984ff2c3..9fe355b9c0 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -85,7 +85,6 @@ def add_common_arguments(p, mask = None): test.add_argument("-f", "--from", default=test_cases[0].get_uuid(), type=str, help="First test UUID to run.") test.add_argument("-t", "--to", default=test_cases[-1].get_uuid(), type=str, help="Last test UUID to run.") test.add_argument("-o", "--only", nargs="+", type=str, default=[], metavar="L", help="Only run tests with UUIDs or hashes L.") - test.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Serial) Override MPI execution binary") test.add_argument("-r", "--relentless", action="store_true", default=False, help="Run all tests, even if multiple fail.") test.add_argument("-a", "--test-all", action="store_true", default=False, help="Run the Post Process Tests too.") test.add_argument("-%", "--percent", type=int, default=100, help="Percentage of tests to run.") @@ -98,29 +97,32 @@ def add_common_arguments(p, mask = None): test_meg.add_argument("--add-new-variables", action="store_true", default=False, help="(Test Generation) If new variables are found in D/ when running tests, add them to the golden files.") test_meg.add_argument("--remove-old-tests", action="store_true", default=False, help="(Test Generation) Delete tests directories that are no longer.") + test.add_argument(metavar="FORWARDED", default=[], dest='--', nargs="*", help="Arguments to forward to the ./mfc.sh run invocations.") + # === RUN === engines = [ e.slug for e in ENGINES ] add_common_arguments(run) - run.add_argument("input", metavar="INPUT", type=str, help="Input file to run.") - run.add_argument("arguments", metavar="ARGUMENTS", nargs='*', type=str, default=[], help="Additional arguments to pass to the case file.") - run.add_argument("-e", "--engine", choices=engines, type=str, default=engines[0], help="Job execution/submission engine choice.") - run.add_argument("-p", "--partition", metavar="PARTITION", type=str, default="", help="(Batch) Partition for job submission.") - run.add_argument("-N", "--nodes", metavar="NODES", type=int, default=1, help="(Batch) Number of nodes.") - run.add_argument("-n", "--tasks-per-node", metavar="TASKS", type=int, default=1, help="Number of tasks per node.") - run.add_argument("-w", "--walltime", metavar="WALLTIME", type=str, default="01:00:00", help="(Batch) Walltime.") - run.add_argument("-a", "--account", metavar="ACCOUNT", type=str, default="", help="(Batch) Account to charge.") - run.add_argument("-@", "--email", metavar="EMAIL", type=str, default="", help="(Batch) Email for job notification.") - run.add_argument("-#", "--name", metavar="NAME", type=str, default="MFC", help="(Batch) Job name.") - run.add_argument("-f", "--flags", metavar="FLAGS", nargs='+', type=str, default=[], help="(Batch) Additional batch options.") - run.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Interactive) Override MPI execution binary") - run.add_argument("-s", "--scratch", action="store_true", default=False, help="Build from scratch.") - run.add_argument("--ncu", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Compute.") - run.add_argument("--nsys", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Systems.") - run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.") - run.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") - run.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.") - run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.") + run.add_argument("input", metavar="INPUT", type=str, help="Input file to run.") + run.add_argument("arguments", metavar="ARGUMENTS", nargs="*", type=str, default=[], help="Additional arguments to pass to the case file.") + run.add_argument("-e", "--engine", choices=engines, type=str, default=engines[0], help="Job execution/submission engine choice.") + run.add_argument("--output-summary", type=str, default=None, help="(Interactive) Output a YAML summary file.") + run.add_argument("-p", "--partition", metavar="PARTITION", type=str, default="", help="(Batch) Partition for job submission.") + run.add_argument("-N", "--nodes", metavar="NODES", type=int, default=1, help="(Batch) Number of nodes.") + run.add_argument("-n", "--tasks-per-node", metavar="TASKS", type=int, default=1, help="Number of tasks per node.") + run.add_argument("-w", "--walltime", metavar="WALLTIME", type=str, default="01:00:00", help="(Batch) Walltime.") + run.add_argument("-a", "--account", metavar="ACCOUNT", type=str, default="", help="(Batch) Account to charge.") + run.add_argument("-@", "--email", metavar="EMAIL", type=str, default="", help="(Batch) Email for job notification.") + run.add_argument("-#", "--name", metavar="NAME", type=str, default="MFC", help="(Batch) Job name.") + run.add_argument("-b", "--binary", choices=binaries, type=str, default=None, help="(Interactive) Override MPI execution binary") + run.add_argument("-s", "--scratch", action="store_true", default=False, help="Build from scratch.") + run.add_argument("--ncu", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Compute.") + run.add_argument("--nsys", nargs=argparse.REMAINDER, type=str, help="Profile with NVIDIA Nsight Systems.") + run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.") + run.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") + run.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.") + run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.") + run.add_argument("-f", "--flags", metavar="FLAGS", dest="--", nargs=argparse.REMAINDER, type=str, default=[], help="(Interactive) Arguments to forward to the MPI invocation.") # === BENCH === add_common_arguments(bench, "t") @@ -129,6 +131,7 @@ def add_common_arguments(p, mask = None): add_common_arguments(count, "g") args: dict = vars(parser.parse_args()) + args["--"] = args.get("--", []) # Add default arguments of other subparsers for name, parser in [("run", run), ("test", test), ("build", build), diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index 4c4347802a..3701a4e89f 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -57,6 +57,25 @@ def system(command: typing.List[str], no_exception: bool = False, exception_text return r.returncode +def system_raw(command: typing.List[str], **kwargs) -> subprocess.CompletedProcess: + cmd = [ str(x) for x in command if not isspace(str(x)) ] + + is_print_visible = kwargs.get("stdout") not in [ + subprocess.DEVNULL, subprocess.PIPE, None + ] + + if is_print_visible: + cons.print(no_indent=True) + cons.print(f"$ {' '.join(cmd)}") + + result = subprocess.run(command, **kwargs, check=False) + + if is_print_visible: + cons.print(no_indent=True) + + return result + + def file_write(filepath: str, content: str): try: with open(filepath, "w") as f: diff --git a/toolchain/mfc/run/engines.py b/toolchain/mfc/run/engines.py index 7ca4c67e5c..988363c926 100644 --- a/toolchain/mfc/run/engines.py +++ b/toolchain/mfc/run/engines.py @@ -81,7 +81,7 @@ def get_args(self) -> str: def get_exec_cmd(self, target: MFCTarget) -> typing.List[str]: cmd = [] if ARG("mpi"): - cmd += [self.mpibin.bin] + self.mpibin.gen_params() + ARG("flags")[:] + cmd += [self.mpibin.bin] + self.mpibin.gen_params() + ARG("--")[:] cmd += profiler_prepend() @@ -89,7 +89,6 @@ def get_exec_cmd(self, target: MFCTarget) -> typing.List[str]: return cmd - def run(self, targets: typing.List[MFCTarget]) -> None: if not self.bKnowWorks: # Fix MFlowCode/MFC#21: Check whether attempting to run a job will hang diff --git a/toolchain/mfc/test/case.py b/toolchain/mfc/test/case.py index aca9cebbd2..b7159be05e 100644 --- a/toolchain/mfc/test/case.py +++ b/toolchain/mfc/test/case.py @@ -106,29 +106,26 @@ def __init__(self, trace: str, mods: dict, ppn: int = None, opt: bool = None) -> def run(self, targets: typing.List[typing.Union[str, MFCTarget]], gpus: typing.Set[int]) -> subprocess.CompletedProcess: if gpus is not None and len(gpus) != 0: - gpus_select = f"--gpus {' '.join([str(_) for _ in gpus])}" + gpus_select = ["--gpus"] + [str(_) for _ in gpus] else: - gpus_select = "" + gpus_select = [] - filepath = f'"{self.get_dirpath()}/case.py"' - tasks = f"-n {self.ppn}" - jobs = f"-j {ARG('jobs')}" if ARG("case_optimization") else "" - binary_option = f"-b {ARG('binary')}" if ARG("binary") is not None else "" - case_optimization = "--case-optimization" if ARG("case_optimization") or self.opt else "--no-build" + filepath = f'{self.get_dirpath()}/case.py' + tasks = ["-n", str(self.ppn)] + jobs = ["-j", str(ARG("jobs"))] if ARG("case_optimization") else [] + case_optimization = ["--case-optimization"] if ARG("case_optimization") or self.opt else ["--no-build"] mfc_script = ".\\mfc.bat" if os.name == 'nt' else "./mfc.sh" target_names = [ get_target(t).name for t in targets ] - command: str = f'''\ - {mfc_script} run {filepath} {tasks} {binary_option} \ - {case_optimization} {jobs} -t {' '.join(target_names)} \ - {gpus_select} 2>&1\ - ''' + command = [ + mfc_script, "run", filepath, *tasks, *case_optimization, + *jobs, "-t", *target_names, *gpus_select, *ARG("--") + ] - return subprocess.run(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True, - shell=True, check=False) + return common.system_raw( + command, text=True, capture_output=True) def get_uuid(self) -> str: return hex(binascii.crc32(hashlib.sha1(str(self.trace).encode()).digest())).upper()[2:].zfill(8)