diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..cc553ca65f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,17 @@ +name: Lint + +on: + push: + + workflow_dispatch: + +jobs: + docs: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Lint + run: ./mfc.sh lint diff --git a/mfc.sh b/mfc.sh index fdb3e59d0e..0b4f8335dd 100755 --- a/mfc.sh +++ b/mfc.sh @@ -439,10 +439,23 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > / fi -# Run the main.py bootstrap script -python3 "$(pwd)/toolchain/mfc.py" "$@" -code=$? +if [ "$1" == "lint" ]; then + shift + + if ! command -v pylint > /dev/null 2>&1; then + error "Couldn't find$MAGENTA pylint$COLOR_RESET. Please ensure it is discoverable." + + exit 1 + fi + + log "(venv) Running$MAGENTA pylint$COLOR_RESET on$MAGENTA MFC$COLOR_RESET." + pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640 toolchain/ +else + # Run the main.py bootstrap script + python3 "$(pwd)/toolchain/mfc.py" "$@" + code=$? +fi # Deactivate the Python virtualenv in case the user "source"'d this script log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment." diff --git a/toolchain/bench.yaml b/toolchain/bench.yaml index 110f4e354f..06df0fcdaf 100644 --- a/toolchain/bench.yaml +++ b/toolchain/bench.yaml @@ -1,3 +1,9 @@ -- name: 1D_bubblescreen - path: examples/1D_bubblescreen/case.py +- name: 3D_shockdroplet + path: examples/3D_shockdroplet/case.py + args: [] +- name: 3D_sphbubcollapse + path: examples/3D_sphbubcollapse/case.py + args: [] +- name: 3D_turb_mixing + path: examples/3D_turb_mixing/case.py args: [] diff --git a/toolchain/mfc.py b/toolchain/mfc.py index c783781b21..1ca38f247a 100644 --- a/toolchain/mfc.py +++ b/toolchain/mfc.py @@ -12,11 +12,11 @@ def __print_greeting(): MFC_LOGO_LINES = MFC_LOGO.splitlines() - max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ]) + max_logo_line_length = max(len(line) for line in MFC_LOGO_LINES) host_line = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]" targets_line = f"[bold]--targets {format_list_to_string(ARG('targets'), 'magenta', 'None')}[/bold]" - help_line = "$ ./mfc.sh \[build, run, test, clean, count, packer] --help" + help_line = "$ ./mfc.sh [build, run, test, clean, count, packer] --help" MFC_SIDEBAR_LINES = [ "", @@ -47,7 +47,7 @@ def __checks(): raise MFCException("CMake is required to build MFC but couldn't be located on your system. Please ensure it installed and discoverable (e.g in your system's $PATH).") -def __run(): +def __run(): {"test": test.test, "run": run.run, "build": build.build, "clean": build.clean, "bench": bench.bench, "count": count.count, "packer": packer.packer @@ -60,7 +60,7 @@ def __run(): state.gARG = args.parse(state.gCFG) lock.switch(state.MFCConfig.from_dict(state.gARG)) - + __print_greeting() __checks() __run() diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index ba870940ab..e563e09cfc 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -3,7 +3,6 @@ from .build import TARGETS, DEFAULT_TARGETS, DEPENDENCY_TARGETS from .common import MFCException, format_list_to_string from .test.test import CASES as TEST_CASES -from .packer import packer def parse(config): from .run.engines import ENGINES @@ -11,7 +10,7 @@ def parse(config): parser = argparse.ArgumentParser( prog="./mfc.sh", - description=f"""\ + description="""\ Welcome to the MFC master script. This tool automates and manages building, testing, \ running, and cleaning of MFC in various configurations on all supported platforms. \ The README documents this tool and its various commands in more detail. To get \ @@ -19,7 +18,7 @@ def parse(config): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parsers = parser.add_subparsers(dest="command") + parsers = parser.add_subparsers(dest="command") run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -27,12 +26,12 @@ def parse(config): bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter) count = parsers.add_parser(name="count", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) packer = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - + packers = packer.add_subparsers(dest="packer") pack = packers.add_parser(name="pack", help="Pack a case into a single file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) pack.add_argument("input", metavar="INPUT", type=str, default="", help="Input file of case to pack.") pack.add_argument("-o", "--output", metavar="OUTPUT", type=str, default=None, help="Base name of output file.") - + compare = packers.add_parser(name="compare", help="Compare two cases.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) compare.add_argument("input1", metavar="INPUT1", type=str, default=None, help="First pack file.") compare.add_argument("input2", metavar="INPUT2", type=str, default=None, help="Second pack file.") @@ -54,7 +53,7 @@ def add_common_arguments(p, mask = None): p.add_argument(f"--no-{f.name}", action="store_false", dest=f.name, help=f"Turn the {f.name} option OFF.") p.set_defaults(**{ f.name: getattr(config, f.name) for f in dataclasses.fields(config) }) - + if "j" not in mask: p.add_argument("-j", "--jobs", metavar="JOBS", type=int, default=1, help="Allows for JOBS concurrent jobs.") @@ -91,7 +90,7 @@ def add_common_arguments(p, mask = None): test.add_argument("-m", "--max-attempts", type=int, default=3, help="Maximum number of attempts to run a test.") test.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.") - + test_meg = test.add_mutually_exclusive_group() test_meg.add_argument("--generate", action="store_true", default=False, help="(Test Generation) Generate golden files.") test_meg.add_argument("--add-new-variables", action="store_true", default=False, help="(Test Generation) If new variables are found in D/ when running tests, add them to the golden files.") @@ -154,7 +153,7 @@ def add_common_arguments(p, mask = None): # build's --case-optimization and --input depend on each other if args["command"] == "build": if (args["input"] is not None) ^ args["case_optimization"] : - raise MFCException(f"./mfc.sh build's --case-optimization requires --input") + raise MFCException("./mfc.sh build's --case-optimization requires --input") # Input files to absolute paths for e in ["input", "input1", "input2"]: diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index d36b756755..759e221404 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -2,9 +2,8 @@ from .printer import cons from .state import ARG, CFG -from .build import PRE_PROCESS, SIMULATION, build_targets +from .build import PRE_PROCESS, SIMULATION from .common import system, MFC_BENCH_FILEPATH, file_load_yaml, file_dump_yaml -from . import sched @dataclasses.dataclass diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 36a678b9bb..c51ff9eb12 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -1,8 +1,7 @@ -import re, os, typing, hashlib, dataclasses +import os, typing, hashlib, dataclasses from .common import MFCException, system, delete_directory, create_directory from .state import ARG, CFG -from .printer import cons from .run import input @dataclasses.dataclass @@ -26,10 +25,10 @@ def compute(self) -> typing.Set: isRequired: bool # Should it always be built? (no matter what -t | --targets is) requires: Dependencies # Build dependencies of the target runOrder: int # For MFC Targets: Order in which targets should logically run - + def __hash__(self) -> int: return hash(self.name) - + # Get path to directory that will store the build files def get_build_dirpath(self) -> str: subdir = 'dependencies' if self.isDependency else CFG().make_slug() @@ -66,7 +65,7 @@ def get_install_dirpath(self) -> str: "install", 'dependencies' if self.isDependency else CFG().make_slug() ]) - + def get_install_binpath(self) -> str: # /install//bin/ return os.sep.join([self.get_install_dirpath(), "bin", self.name]) @@ -104,17 +103,17 @@ def configure(self): build_dirpath = self.get_build_dirpath() cmake_dirpath = self.get_cmake_dirpath() install_dirpath = self.get_install_dirpath() - + install_prefixes = ';'.join([install_dirpath, get_dependency_install_dirpath()]) flags: list = self.flags.copy() + [ # Disable CMake warnings intended for developers (us). # See: https://cmake.org/cmake/help/latest/manual/cmake.1.html. - f"-Wno-dev", + "-Wno-dev", # Save a compile_commands.json file with the compile commands used to # build the configured targets. This is mostly useful for debugging. # See: https://cmake.org/cmake/help/latest/variable/CMAKE_EXPORT_COMPILE_COMMANDS.html. - f"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", # Set build type (e.g Debug, Release, etc.). # See: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html f"-DCMAKE_BUILD_TYPE={'Debug' if ARG('debug') else 'Release'}", @@ -160,7 +159,7 @@ def install(self): install = ["cmake", "--install", self.get_build_dirpath()] system(install, exception_text=f"Failed to install the [bold magenta]{self.name}[/bold magenta] target.") - + def clean(self): build_dirpath = self.get_build_dirpath() @@ -196,7 +195,7 @@ def clean(self): def get_target(target: typing.Union[str, MFCTarget]) -> MFCTarget: if isinstance(target, MFCTarget): return target - + if target in TARGET_MAP: return TARGET_MAP[target] @@ -218,7 +217,7 @@ def build_target(target: typing.Union[MFCTarget, str], history: typing.Set[str] history = set() t = get_target(target) - + if t.name in history or not t.is_buildable(): return @@ -235,7 +234,7 @@ def build_target(target: typing.Union[MFCTarget, str], history: typing.Set[str] def build_targets(targets: typing.Iterable[typing.Union[MFCTarget, str]], history: typing.Set[str] = None): if history is None: history = set() - + for target in list(REQUIRED_TARGETS) + targets: build_target(target, history) @@ -245,7 +244,7 @@ def clean_target(target: typing.Union[MFCTarget, str], history: typing.Set[str] history = set() t = get_target(target) - + if t.name in history or not t.is_buildable(): return diff --git a/toolchain/mfc/case.py b/toolchain/mfc/case.py index d6a11c74a6..8e195ce43c 100644 --- a/toolchain/mfc/case.py +++ b/toolchain/mfc/case.py @@ -6,7 +6,7 @@ class Case: def __init__(self, params: dict) -> None: self.params = copy.deepcopy(params) - + def get_parameters(self) -> str: return self.params.keys() diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index 866c5d00ac..33f4dbc205 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -1,6 +1,4 @@ -import os, yaml, typing, shutil, subprocess, dataclasses - -from datetime import datetime +import os, yaml, typing, shutil, subprocess from .printer import cons @@ -13,7 +11,7 @@ MFC_LOCK_FILEPATH = abspath(f"{MFC_SUBDIR}/lock.yaml") MFC_BENCH_FILEPATH = abspath(f"{MFC_ROOTDIR}/toolchain/bench.yaml") -MFC_LOGO = f""" +MFC_LOGO = """ .=++*: -+*+=. :+ -*- == =* . :*+ == ++ .+- @@ -40,7 +38,7 @@ def system(command: typing.List[str], no_exception: bool = False, exception_text cons.print(no_indent=True) cons.print(f"$ {' '.join(cmd)}") - + if stdout != subprocess.DEVNULL: cons.print(no_indent=True) @@ -121,7 +119,7 @@ def get_program_output(arguments: typing.List[str] = None, cwd=None): return (proc.communicate()[0].decode(), proc.returncode) -def get_py_program_output(filepath: str, arguments: typing.List[str] = None): +def get_py_program_output(filepath: str, arguments: typing.List[str] = None): dirpath = os.path.abspath (os.path.dirname(filepath)) filename = os.path.basename(filepath) @@ -156,12 +154,12 @@ def does_command_exist(s: str) -> bool: def format_list_to_string(arr: list, item_style=None, empty=None): if empty is None: empty = "nothing" - + pre, post = "", "" if item_style is not None: pre = f"[{item_style}]" post = f"[/{item_style}]" - + if len(arr) == 0: return f"{pre}{empty}{post}" @@ -189,6 +187,7 @@ def find(predicate, arr: list): return None, None +# pylint: disable=redefined-builtin def quit(sig): os.kill(os.getpid(), sig) @@ -212,10 +211,10 @@ def get_loaded_modules() -> typing.List[str]: def is_number(x: str) -> bool: if x is None: return False - + if isinstance(x, int) or isinstance(x, float): return True - + try: float(x) return True diff --git a/toolchain/mfc/count.py b/toolchain/mfc/count.py index 4ffabdf289..7293385430 100644 --- a/toolchain/mfc/count.py +++ b/toolchain/mfc/count.py @@ -25,17 +25,17 @@ def count(): cons.print(f"[bold]Counting lines of code in {target_str_list}[/bold] (excluding whitespace lines)") cons.indent() - + total = 0 for codedir in ['common'] + ARG("targets"): - dirfiles, dircount = handle_dir(os.path.join(MFC_ROOTDIR, 'src', codedir)) + dirfiles, dircount = handle_dir(os.path.join(MFC_ROOTDIR, 'src', codedir)) table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) table.add_column(f"File (in [magenta]{codedir}[/magenta])", justify="left") table.add_column(f"Lines ([cyan]{dircount}[/cyan])", justify="right") - + for filepath, count in dirfiles: table.add_row(f"{os.path.basename(filepath)}", f"[bold cyan]{count}[/bold cyan]") - + total += dircount cons.raw.print(table) diff --git a/toolchain/mfc/lock.py b/toolchain/mfc/lock.py index f60890250c..58b9d95a9a 100644 --- a/toolchain/mfc/lock.py +++ b/toolchain/mfc/lock.py @@ -19,7 +19,7 @@ class MFCLockData: def init(): global data - + if not os.path.exists(common.MFC_LOCK_FILEPATH): config = MFCConfig() data = MFCLockData(config, MFC_LOCK_CURRENT_VERSION) @@ -33,9 +33,9 @@ def init(): def load(): global data - + d = common.file_load_yaml(common.MFC_LOCK_FILEPATH) - + # 0 is the default version in order to accommodate versions of mfc.sh # prior to the introduction of the "version" attribute to the lock file. @@ -52,13 +52,13 @@ def load(): def write(): global data - + common.file_dump_yaml(common.MFC_LOCK_FILEPATH, dataclasses.asdict(data)) def switch(to: MFCConfig): global data - + if to == data.config: return diff --git a/toolchain/mfc/packer/errors.py b/toolchain/mfc/packer/errors.py index 63a68e98da..66ab8d4463 100644 --- a/toolchain/mfc/packer/errors.py +++ b/toolchain/mfc/packer/errors.py @@ -49,4 +49,4 @@ def push(self, error: Error) -> None: self.count += 1 def __repr__(self) -> str: - return self.get().__repr__() \ No newline at end of file + return self.get().__repr__() diff --git a/toolchain/mfc/packer/pack.py b/toolchain/mfc/packer/pack.py index 6e232854b7..f2a799f182 100644 --- a/toolchain/mfc/packer/pack.py +++ b/toolchain/mfc/packer/pack.py @@ -3,7 +3,6 @@ from .. import common from ..build import get_configured_targets from ..state import CFG -from ..common import MFCException from datetime import datetime from pathlib import Path @@ -40,13 +39,13 @@ def set(self, entry: PackEntry): def save(self, filepath: str): if filepath.endswith(".py"): filepath = os.path.dirname(filepath) - + if os.path.isdir(filepath): filepath = os.path.join(filepath, "pack.txt") - + if not filepath.endswith(".txt"): filepath += ".txt" - + common.file_write(filepath, '\n'.join([ str(e) for e in sorted(self.entries.values(), key=lambda x: x.filepath) ])) metadata = f"""\ @@ -91,7 +90,7 @@ def has_NaNs(self) -> bool: def load(filepath: str) -> Pack: if not os.path.isfile(filepath): filepath = os.path.join(filepath, "pack.txt") - + entries: typing.List[PackEntry] = [] for line in common.file_read(filepath).splitlines(): diff --git a/toolchain/mfc/packer/packer.py b/toolchain/mfc/packer/packer.py index 1ac1bc108f..462b8fc022 100644 --- a/toolchain/mfc/packer/packer.py +++ b/toolchain/mfc/packer/packer.py @@ -13,12 +13,12 @@ def load(packpath: str) -> _pack.Pack: def pack(casepath: str, packpath: str = None) -> typing.Tuple[_pack.Pack, str]: if packpath is None: packpath = casepath - + p, err = _pack.compile(casepath) - + if err is not None: return None, err - + p.save(packpath) return p, None @@ -35,19 +35,19 @@ def packer(): if ARG("packer") == "pack": if not os.path.isdir(ARG("input")): ARGS()["input"] = os.path.dirname(ARG("input")) - + out_dir = os.path.sep.join([ARG("input"), ARG("output")]) if ARG("output") is not None else None p, err = pack(ARG("input"), out_dir) if err is not None: raise MFCException(err) - elif ARG("packer") == "compare": + elif ARG("packer") == "compare": cons.print(f"Comparing [magenta]{os.path.relpath(ARG('input1'), os.getcwd())}[/magenta] to [magenta]{os.path.relpath(ARG('input1'), os.getcwd())}[/magenta]:") - + cons.indent() cons.print() - + tol = packtol.Tolerance(ARG("abstol"), ARG("reltol")) - + err, msg = compare(ARG("input1"), ARG("input2"), tol) if msg is not None: cons.print(f"[bold red]ERROR[/bold red]: The two packs are not within tolerance ({tol}).") @@ -58,6 +58,6 @@ def packer(): cons.print() cons.unindent() - + else: raise MFCException(f"Unknown packer command: {ARG('packer')}") diff --git a/toolchain/mfc/packer/tol.py b/toolchain/mfc/packer/tol.py index 20b6a7ec45..f2b985fde3 100644 --- a/toolchain/mfc/packer/tol.py +++ b/toolchain/mfc/packer/tol.py @@ -1,7 +1,5 @@ import math, typing -from ..common import MFCException - from .pack import Pack from .errors import compute_error, AverageError, Error diff --git a/toolchain/mfc/printer.py b/toolchain/mfc/printer.py index 91eda7f1e1..e4a3f47451 100644 --- a/toolchain/mfc/printer.py +++ b/toolchain/mfc/printer.py @@ -13,20 +13,20 @@ def reset(self): def indent(self, msg: str = None): msg = msg if msg is not None else " " - + self.stack.append(msg) def unindent(self, times: int = None): if times == None: times = 1 - + for _ in range(times): self.stack.pop() def print(self, msg: typing.Any = None, no_indent: bool = False, *args, **kwargs): if msg is None: msg = "" - + if no_indent: self.raw.print(str(msg), *args, **kwargs) else: diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index 24ef70b82a..1098e9c107 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -159,5 +159,5 @@ def get_input_dict_keys(target_name: str) -> list: if not ARG("case_optimization") or target_name != "simulation": return result - + return [ x for x in result if x not in CASE_OPTIMIZATION ] diff --git a/toolchain/mfc/run/engines.py b/toolchain/mfc/run/engines.py index d7a6dbe53b..ebd37714e4 100644 --- a/toolchain/mfc/run/engines.py +++ b/toolchain/mfc/run/engines.py @@ -1,4 +1,4 @@ -import re, os, time, copy, queue, typing, datetime, subprocess, dataclasses, multiprocessing +import re, os, time, copy, typing, datetime, subprocess, dataclasses, multiprocessing from ..state import ARG, ARGS from ..printer import cons @@ -119,7 +119,7 @@ def run(self, targets: typing.List[MFCTarget]) -> None: self.bKnowWorks = result.returncode == 0 if not self.bKnowWorks: - error_txt = f"""\ + error_txt = """\ MFC's [bold magenta]syscheck[/bold magenta] (system check) failed to run successfully. Please review the output bellow and ensure that your system is configured correctly: @@ -147,13 +147,13 @@ def run(self, targets: typing.List[MFCTarget]) -> None: if not ARG("dry_run"): start_time = time.monotonic() - + env = os.environ.copy() if ARG('gpus') is not None: env['CUDA_VISIBLE_DEVICES'] = ','.join([str(_) for _ in ARG('gpus')]) system( - self.get_exec_cmd(target), cwd=self.input.case_dirpath, + self.get_exec_cmd(target), cwd=self.input.case_dirpath, env=env ) end_time = time.monotonic() @@ -212,7 +212,7 @@ def __get_batch_filepath(self): ])) def __generate_prologue(self, system: queues.QueueSystem) -> str: - modules = f"" + modules = "" if does_system_use_modules(): modules = f"""\ diff --git a/toolchain/mfc/run/input.py b/toolchain/mfc/run/input.py index 2037998f1c..25593cb3ab 100644 --- a/toolchain/mfc/run/input.py +++ b/toolchain/mfc/run/input.py @@ -15,24 +15,24 @@ class MFCInputFile: filename: str case_dirpath: str case_dict: dict - + def __get_ndims(self) -> int: return 1 + min(int(self.case_dict.get("n", 0)), 1) + min(int(self.case_dict.get("p", 0)), 1) - + def __is_ic_analytical(self, key: str, val: str) -> bool: if common.is_number(val) or not isinstance(val, str): return False - + for array in QPVF_IDX_VARS.keys(): if re.match(fr'^patch_icpp\([0-9]+\)%{array}', key): return True - + return False def generate_inp(self, target) -> None: cons.print(f"Generating [magenta]{target.name}.inp[/magenta].") cons.indent() - + MASTER_KEYS: list = case_dicts.get_input_dict_keys(target.name) ignored = [] @@ -45,12 +45,11 @@ def generate_inp(self, target) -> None: dict_str += f" {key} = 0d0\n" ignored.append(key) continue - + if not isinstance(val, str) or len(val) == 1: dict_str += f"{key} = {val}\n" else: dict_str += f"{key} = '{val}'\n" - continue else: ignored.append(key) @@ -58,12 +57,12 @@ def generate_inp(self, target) -> None: raise common.MFCException(f"MFCInputFile::dump: Case parameter '{key}' is not used by any MFC code. Please check your spelling or add it as a new parameter.") cons.print(f"[yellow]INFO:[/yellow] Forwarded {len(self.case_dict)-len(ignored)}/{len(self.case_dict)} parameters.") - + contents = f"&user_inputs\n{dict_str}&end/\n" # Save .inp input file common.file_write(f"{self.case_dirpath}/{target.name}.inp", contents) - + cons.unindent() def __save_fpp(self, target, contents: str) -> None: @@ -75,8 +74,8 @@ def __contents_equal(str_a: str, str_b: str) -> bool: inc_dir = os.path.join(target.get_build_dirpath(), "include") common.create_directory(inc_dir) - - fpp_path = os.path.join(inc_dir, f"case.fpp") + + fpp_path = os.path.join(inc_dir, "case.fpp") opt_fpp = open(fpp_path).read() if os.path.exists(fpp_path) else "" if __contents_equal(contents, opt_fpp): @@ -84,7 +83,7 @@ def __contents_equal(str_a: str, str_b: str) -> bool: return cons.print("[yellow]INFO:[/yellow] Writing a custom case.fpp file: --case-optimization configuration has changed.") - common.file_write(fpp_path, contents) + common.file_write(fpp_path, contents) def __get_pre_fpp(self) -> str: DATA = { @@ -92,38 +91,38 @@ def __get_pre_fpp(self) -> str: 2: {'ptypes': [2, 3, 4, 5, 6, 7, 17, 18, 21], 'sf_idx': 'i, j, 0'}, 3: {'ptypes': [8, 9, 10, 11, 12, 13, 14, 19, 21], 'sf_idx': 'i, j, k'} }[self.__get_ndims()] - + patches = {} - for key, val in self.case_dict.items(): + for key, val in self.case_dict.items(): if not self.__is_ic_analytical(key, val): continue - + patch_id = re.search(r'[0-9]+', key).group(0) - + if patch_id not in patches: patches[patch_id] = [] - + patches[patch_id].append((key, val)) srcs = [] - + for pid, items in patches.items(): ptype = self.case_dict[f"patch_icpp({pid})%geometry"] - + if ptype not in DATA['ptypes']: raise common.MFCException(f"Patch #{pid} of type {ptype} cannot be analytically defined.") def rhs_replace(match): return { 'x': 'x_cc(i)', 'y': 'y_cc(j)', 'z': 'z_cc(k)', - + 'xc': f'patch_icpp({pid})%x_centroid', 'yc': f'patch_icpp({pid})%y_centroid', 'zc': f'patch_icpp({pid})%z_centroid', 'lx': f'patch_icpp({pid})%length_x', 'ly': f'patch_icpp({pid})%length_y', 'lz': f'patch_icpp({pid})%length_z', - + 'r': f'patch_icpp({pid})%radius', 'eps': f'patch_icpp({pid})%epsilon', 'beta': f'patch_icpp({pid})%beta', 'tau_e': f'patch_icpp({pid})%tau_e', 'radii': f'patch_icpp({pid})%radii', - + 'e' : f'{math.e}', 'pi': f'{math.pi}', }.get(match.group(), match.group()) @@ -132,12 +131,12 @@ def rhs_replace(match): varname = re.findall(r"[a-zA-Z][a-zA-Z0-9_]*", attribute)[1] qpvf_idx_var = QPVF_IDX_VARS[varname] qpvf_idx_offset = "" - + if len(re.findall(r"[0-9]+", attribute)) == 2: idx = int(re.findall(r'[0-9]+', attribute)[1]) - 1 if idx != 0: qpvf_idx_offset = f" + {idx}" - + sf_idx = DATA['sf_idx'] cons.print(f"[yellow]INFO:[/yellow] {self.__get_ndims()}D Analytical Patch #{pid}: Code generation for [magenta]{varname}[/magenta]...") @@ -168,7 +167,7 @@ def rhs_replace(match): def __get_sim_fpp(self) -> str: if ARG("case_optimization"): cons.print("[yellow]INFO:[/yellow] Case optimization is enabled.") - + nterms = -100 bubble_model = int(self.case_dict.get("bubble_model", "-100")) @@ -221,7 +220,7 @@ def generate_fpp(self, target) -> None: def generate(self, target) -> None: self.generate_inp(target) cons.print() - self.generate_fpp(target) + self.generate_fpp(target) # Load the input file @@ -261,7 +260,7 @@ def load(empty_data: dict = None) -> MFCInputFile: raise common.MFCException(f"Input file {filename} did not produce valid JSON. It should only print the case dictionary.\n\n{exc}\n") load.CACHED_MFCInputFile = MFCInputFile(filename, dirpath, dictionary) - + return load.CACHED_MFCInputFile diff --git a/toolchain/mfc/run/queues.py b/toolchain/mfc/run/queues.py index f9e23c2117..dc7d54b063 100644 --- a/toolchain/mfc/run/queues.py +++ b/toolchain/mfc/run/queues.py @@ -44,7 +44,7 @@ def gen_submit_cmd(self, filename: str) -> None: cmd = ["bsub"] if ARG("wait"): - cmd += ["--wait"] + cmd += ["--wait"] return cmd + [filename] @@ -58,7 +58,7 @@ def is_active(self) -> bool: def gen_submit_cmd(self, filename: str) -> None: cmd = ["sbatch"] - + if ARG("wait"): cmd += ["--wait"] @@ -72,4 +72,4 @@ def get_system() -> QueueSystem: if system.is_active(): return system - raise common.MFCException(f"Failed to detect a queue system.") + raise common.MFCException("Failed to detect a queue system.") diff --git a/toolchain/mfc/run/run.py b/toolchain/mfc/run/run.py index ac0d86803d..f16c27c8f9 100644 --- a/toolchain/mfc/run/run.py +++ b/toolchain/mfc/run/run.py @@ -24,12 +24,12 @@ def validate_job_options() -> None: raise MFCException(f'RUN: {ARG("email")} is not a valid e-mail address.') -def run_targets(targets: typing.List[MFCTarget]): +def run_targets(targets: typing.List[MFCTarget]): cons.print("[bold]Run[/bold]") cons.indent() if len(targets) == 0: - cons.print(f"> No target selected.") + cons.print("> No target selected.") return input_file = input.load() @@ -37,7 +37,7 @@ def run_targets(targets: typing.List[MFCTarget]): engine = engines.get_engine(ARG("engine")) engine.init(input_file) - cons.print(f"Configuration:") + cons.print("Configuration:") cons.indent() cons.print(f"""\ Input {ARG('input')} diff --git a/toolchain/mfc/sched.py b/toolchain/mfc/sched.py index f1258ddab3..14cb945bc2 100644 --- a/toolchain/mfc/sched.py +++ b/toolchain/mfc/sched.py @@ -43,7 +43,7 @@ def sched(tasks: typing.List[Task], nThreads: int, devices: typing.Set[int] = No def join_first_dead_thread(progress, complete_tracker) -> None: nonlocal threads, nAvailable - + for threadID, threadHolder in enumerate(threads): if not threadHolder.thread.is_alive(): if threadHolder.thread.exc != None: @@ -90,7 +90,7 @@ def join_first_dead_thread(progress, complete_tracker) -> None: device = min(sched.LOAD.items(), key=lambda x: x[1])[0] sched.LOAD[device] += task.load / task.ppn use_devices.add(device) - + nAvailable -= task.ppn thread = WorkerThread(target=task.func, args=tuple(task.args) + (use_devices,)) diff --git a/toolchain/mfc/state.py b/toolchain/mfc/state.py index 6768329a44..1998800870 100644 --- a/toolchain/mfc/state.py +++ b/toolchain/mfc/state.py @@ -40,7 +40,7 @@ def __eq__(self, other) -> bool: def __str__(self) -> str: """ Returns a string like "mpi=No & gpu=No & debug=No" """ - + return ' & '.join([ f"{k}={'Yes' if v else 'No'}" for k, v in self.items() ]) diff --git a/toolchain/mfc/test/case.py b/toolchain/mfc/test/case.py index 19a63bedcc..3fa7c92135 100644 --- a/toolchain/mfc/test/case.py +++ b/toolchain/mfc/test/case.py @@ -107,15 +107,15 @@ def run(self, targets: typing.List[typing.Union[str, MFCTarget]], gpus: typing.S gpus_select = f"--gpus {' '.join([str(_) for _ in gpus])}" else: gpus_select = "" - + filepath = f'"{self.get_dirpath()}/case.py"' tasks = f"-n {self.ppn}" jobs = f"-j {ARG('jobs')}" if ARG("case_optimization") else "" binary_option = f"-b {ARG('binary')}" if ARG("binary") is not None else "" case_optimization = "--case-optimization" if ARG("case_optimization") or self.opt else "--no-build" - + mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh" - + target_names = [ get_target(t).name for t in targets ] command: str = f'''\ diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index cfe9fbd7f6..db2a5f8066 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -2,13 +2,12 @@ from .. import common from .case import TestCase, create_case, CaseGeneratorStack -from ..printer import cons def get_bc_mods(bc: int, dimInfo): params = {} for dimCmp in dimInfo[0]: params.update({f'bc_{dimCmp}%beg': bc, f'bc_{dimCmp}%end': bc}) - + return params @@ -103,7 +102,7 @@ def alter_riemann_solvers(num_fluids): cases.append(create_case(stack, "mixture_err", {'mixture_err': 'T'})) cases.append(create_case(stack, "avg_state=1", {'avg_state': '1'})) cases.append(create_case(stack, "wave_speeds=2", {'wave_speeds': '2'})) - + if riemann_solver == 2: cases.append(create_case(stack, "model_eqns=3", {'model_eqns': 3})) @@ -131,35 +130,35 @@ def alter_num_fluids(dimInfo, dimParams): alter_riemann_solvers(num_fluids) if num_fluids == 1: - stack.push(f"Viscous", { + stack.push("Viscous", { 'fluid_pp(1)%Re(1)' : 0.0001, 'dt' : 1e-11, 'patch_icpp(1)%vel(1)': 1.0}) - cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) + cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) cases.append(create_case(stack, "weno_Re_flux", {'weno_Re_flux': 'T'})) for weno_Re_flux in ['T']: - stack.push(f"weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) + stack.push("weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) cases.append(create_case(stack, "weno_avg", {'weno_avg': 'T'})) stack.pop() stack.pop() if num_fluids == 2: - stack.push(f"Viscous", { + stack.push("Viscous", { 'fluid_pp(1)%Re(1)' : 0.001, 'fluid_pp(1)%Re(2)' : 0.001, 'fluid_pp(2)%Re(1)' : 0.001, 'fluid_pp(2)%Re(2)' : 0.001, 'dt' : 1e-11, 'patch_icpp(1)%vel(1)': 1.0}) - cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) + cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) cases.append(create_case(stack, "weno_Re_flux", {'weno_Re_flux': 'T'})) for weno_Re_flux in ['T']: - stack.push(f"weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) + stack.push("weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) cases.append(create_case(stack, "weno_avg", {'weno_avg': 'T'})) stack.pop() stack.pop() stack.pop() - stack.pop() + stack.pop() def alter_2d(dimInfo, dimParams): stack.push("Axisymmetric", { @@ -174,14 +173,14 @@ def alter_2d(dimInfo, dimParams): cases.append(create_case(stack, "model_eqns=2", {'model_eqns': 2})) cases.append(create_case(stack, "model_eqns=3", {'model_eqns': 3})) - stack.push(f"Viscous", { + stack.push("Viscous", { 'fluid_pp(1)%Re(1)' : 0.0001, 'fluid_pp(1)%Re(2)' : 0.0001, 'fluid_pp(2)%Re(1)' : 0.0001, 'fluid_pp(2)%Re(2)' : 0.0001, 'dt' : 1e-11}) - cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) + cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) cases.append(create_case(stack, "weno_Re_flux", {'weno_Re_flux': 'T'})) for weno_Re_flux in ['T']: - stack.push(f"weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) + stack.push("weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) cases.append(create_case(stack, "weno_avg", {'weno_avg': 'T'})) stack.pop() @@ -189,7 +188,7 @@ def alter_2d(dimInfo, dimParams): stack.pop() def alter_3d(dimInfo, dimParams): - stack.push(f"Cylindrical", { + stack.push("Cylindrical", { 'bc_y%beg': -14, 'bc_z%beg': -1, 'bc_z%end': -1, 'cyl_coord': 'T', 'x_domain%beg': 0.E+00, 'x_domain%end': 5.E+00, 'y_domain%beg': 0.E+00, 'y_domain%end': 1.E+00, 'z_domain%beg': 0.E+00, 'z_domain%end' : 2.0*3.141592653589793E+00, 'm': 29, 'n': 29, 'p': 29, @@ -212,15 +211,15 @@ def alter_3d(dimInfo, dimParams): cases.append(create_case(stack, "model_eqns=2", {'model_eqns': 2})) - stack.push(f"Viscous", { + stack.push("Viscous", { 'fluid_pp(1)%Re(1)' : 0.0001, 'fluid_pp(1)%Re(2)' : 0.0001, 'fluid_pp(2)%Re(1)' : 0.0001, 'fluid_pp(2)%Re(2)' : 0.0001, 'dt' : 1e-11 - }) + }) - cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) + cases.append(create_case(stack, "", {'weno_Re_flux': 'F'})) cases.append(create_case(stack, "weno_Re_flux", {'weno_Re_flux': 'T'})) for weno_Re_flux in ['T']: - stack.push(f"weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) + stack.push("weno_Re_flux" if weno_Re_flux == 'T' else '', {'weno_Re_flux' : 'T'}) cases.append(create_case(stack, "weno_avg", {'weno_avg': 'T'})) stack.pop() @@ -229,15 +228,15 @@ def alter_3d(dimInfo, dimParams): def alter_ppn(dimInfo, dimParams): if len(dimInfo[0]) == 3: - cases.append(create_case(stack, f'2 MPI Ranks', {'m': 29, 'n': 29, 'p': 49}, ppn=2)) + cases.append(create_case(stack, '2 MPI Ranks', {'m': 29, 'n': 29, 'p': 49}, ppn=2)) else: - cases.append(create_case(stack, f'2 MPI Ranks', {}, ppn=2)) + cases.append(create_case(stack, '2 MPI Ranks', {}, ppn=2)) def alter_bubbles(dimInfo, dimParams): if len(dimInfo[0]) > 0: - stack.push(f"Bubbles", {"bubbles": 'T'}) + stack.push("Bubbles", {"bubbles": 'T'}) - stack.push(f'', { + stack.push('', { 'nb' : 3, 'fluid_pp(1)%gamma' : 0.16, 'fluid_pp(1)%pi_inf': 3515.0, 'fluid_pp(2)%gamma': 2.5, 'fluid_pp(2)%pi_inf': 0.0, 'fluid_pp(1)%mul0' : 0.001002, 'fluid_pp(1)%ss' : 0.07275,'fluid_pp(1)%pv' : 2338.8,'fluid_pp(1)%gamma_v' : 1.33, @@ -249,7 +248,7 @@ def alter_bubbles(dimInfo, dimParams): 'patch_icpp(3)%pres': 1.0 }) - stack.push(f"Monopole", {"Monopole": 'T'}) + stack.push("Monopole", {"Monopole": 'T'}) if len(dimInfo[0]) >= 2: stack.push("", {'Mono(1)%loc(2)': 0.5}) @@ -258,7 +257,7 @@ def alter_bubbles(dimInfo, dimParams): stack.push("", {'Mono(1)%loc(3)': 0.5, 'Mono(1)%support': 3}) for polytropic in ['T', 'F']: - stack.push(f"Polytropic" if polytropic == 'T' else '', {'polytropic' : polytropic}) + stack.push("Polytropic" if polytropic == 'T' else '', {'polytropic' : polytropic}) for bubble_model in [3, 2]: stack.push(f"bubble_model={bubble_model}", {'bubble_model' : bubble_model}) @@ -273,11 +272,11 @@ def alter_bubbles(dimInfo, dimParams): stack.push('', {'polytropic': 'T', 'bubble_model': 2}) cases.append(create_case(stack, 'nb=1', {'nb': 1})) - stack.push(f"QBMM", {'qbmm': 'T'}) + stack.push("QBMM", {'qbmm': 'T'}) cases.append(create_case(stack, '', {})) - stack.push(f"Non-polytropic", {'polytropic': 'F'}) + stack.push("Non-polytropic", {'polytropic': 'F'}) cases.append(create_case(stack, '', {})) stack.pop() @@ -285,7 +284,7 @@ def alter_bubbles(dimInfo, dimParams): stack.push('bubble_model=3', {'bubble_model': 3, 'polytropic': 'T'}) cases.append(create_case(stack, '', {})) - stack.push(f'Non-polytropic', { 'polytropic': 'F'}) + stack.push('Non-polytropic', { 'polytropic': 'F'}) cases.append(create_case(stack, '', {})) for i in range(7): @@ -352,10 +351,10 @@ def alter_hypoelasticity(dimInfo, dimParams): def alter_viscosity(dimInfo, dimParams): # Viscosity & bubbles checks if len(dimInfo[0]) > 0: - stack.push(f"Viscosity -> Bubbles", + stack.push("Viscosity -> Bubbles", {"fluid_pp(1)%Re(1)": 50, "bubbles": 'T'}) - stack.push(f'', { + stack.push('', { 'nb' : 1, 'fluid_pp(1)%gamma' : 0.16, 'fluid_pp(1)%pi_inf': 3515.0, 'fluid_pp(2)%gamma': 2.5, 'fluid_pp(2)%pi_inf': 0.0, 'fluid_pp(1)%mul0' : 0.001002, 'fluid_pp(1)%ss' : 0.07275,'fluid_pp(1)%pv' : 2338.8,'fluid_pp(1)%gamma_v' : 1.33, @@ -368,7 +367,7 @@ def alter_viscosity(dimInfo, dimParams): }) for polytropic in ['T', 'F']: - stack.push(f"Polytropic" if polytropic == 'T' else '', {'polytropic' : polytropic}) + stack.push("Polytropic" if polytropic == 'T' else '', {'polytropic' : polytropic}) for bubble_model in [3, 2]: stack.push(f"bubble_model={bubble_model}", {'bubble_model' : bubble_model}) @@ -383,7 +382,7 @@ def alter_viscosity(dimInfo, dimParams): stack.push('', {'polytropic': 'T', 'bubble_model': 2}) cases.append(create_case(stack, 'nb=1', {'nb': 1})) - stack.push(f"QBMM", {'qbmm': 'T'}) + stack.push("QBMM", {'qbmm': 'T'}) cases.append(create_case(stack, '', {})) stack.push('bubble_model=3', {'bubble_model': 3}) @@ -409,7 +408,7 @@ def foreach_dimension(): alter_viscosity(dimInfo, dimParams) stack.pop() stack.pop() - + foreach_dimension() # Sanity Check 1 diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 7dc19d73d9..08ff522b05 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -21,7 +21,7 @@ def __filter(): global CASES - + # Check "--from" and "--to" exist and are in the right order bFoundFrom, bFoundTo = (False, False) from_i = -1 @@ -64,7 +64,7 @@ def __filter(): def test(): global CASES, nFAIL - + # Delete UUIDs that are not in the list of CASES from tests/ if ARG("remove_old_tests"): dir_uuids = set(os.listdir(common.MFC_TESTDIR)) @@ -73,7 +73,7 @@ def test(): for old_uuid in dir_uuids - new_uuids: cons.print(f"[bold red]Deleting:[/bold red] {old_uuid}") common.delete_directory(f"{common.MFC_TESTDIR}/{old_uuid}") - + return __filter() @@ -101,15 +101,15 @@ def test(): if len(ARG("only")) > 0: range_str = "Only " + format_list_to_string(ARG("only"), "bold magenta", "Nothing to run") - + cons.print(f"[bold]Test {format_list_to_string([ x.name for x in codes ], 'magenta')}[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})") cons.indent() # Run CASES with multiple threads (if available) cons.print() - cons.print(f" tests/[bold magenta]UUID[/bold magenta] (s) Summary") + cons.print(" tests/[bold magenta]UUID[/bold magenta] (s) Summary") cons.print() - + # Select the correct number of threads to use to launch test CASES # We can't use ARG("jobs") when the --case-optimization option is set # because running a test case may cause it to rebuild, and thus @@ -121,10 +121,10 @@ def test(): cons.print() if nFAIL == 0: - cons.print(f"Tested Simulation [bold green]✓[/bold green]") + cons.print("Tested Simulation [bold green]✓[/bold green]") else: if nFAIL == 1: - raise MFCException(f"Testing: There was [bold red]1[/bold red] failure.") + raise MFCException("Testing: There was [bold red]1[/bold red] failure.") else: raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.") @@ -139,7 +139,7 @@ def _handle_case(test: TestCase, devices: typing.Set[int]): elif test.params.get("bubbles", 'F') == 'T': tol = 1e-10 elif test.params.get("hypoelasticity", 'F') == 'T': - tol = 1e-7 + tol = 1e-7 else: tol = 1e-12 @@ -195,13 +195,13 @@ def _handle_case(test: TestCase, devices: typing.Set[int]): silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p0', f'{t_step}.silo') if not os.path.exists(silo_filepath): silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p_all', 'p0', f'{t_step}.silo') - + h5dump = f"{HDF5.get_install_dirpath()}/bin/h5dump" if ARG("no_hdf5"): if not does_command_exist("h5dump"): raise MFCException("--no-hdf5 was specified and h5dump couldn't be found.") - + h5dump = shutil.which("h5dump") output, err = get_program_output([h5dump, silo_filepath]) @@ -225,7 +225,7 @@ def _handle_case(test: TestCase, devices: typing.Set[int]): def handle_case(test: TestCase, devices: typing.Set[int]): global nFAIL - + nAttempts = 0 while True: diff --git a/toolchain/requirements.txt b/toolchain/requirements.txt index afee88b5af..6d5ade73c2 100644 --- a/toolchain/requirements.txt +++ b/toolchain/requirements.txt @@ -3,5 +3,6 @@ fypp wheel typing PyYAML +pylint argparse dataclasses