MFlowCode#230: Linting the toolchain/

.github/workflows/lint.yml

@@ -0,0 +1,17 @@
+name: Lint
+
+on:
+  push:
+
+  workflow_dispatch:
+
+jobs:
+  docs:
+    name:    Lint
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Lint
+      run: ./mfc.sh lint

mfc.sh

@@ -439,10 +439,23 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /
 fi
 
 
-# Run the main.py bootstrap script
-python3 "$(pwd)/toolchain/mfc.py" "$@"
-code=$?
+if [ "$1" == "lint" ]; then
+    shift
+
+    if ! command -v pylint > /dev/null 2>&1; then
+        error "Couldn't find$MAGENTA pylint$COLOR_RESET. Please ensure it is discoverable."
+
+        exit 1
+    fi
+
+    log "(venv) Running$MAGENTA pylint$COLOR_RESET on$MAGENTA MFC$COLOR_RESET."
 
+    pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640 toolchain/
+else
+    # Run the main.py bootstrap script
+    python3 "$(pwd)/toolchain/mfc.py" "$@"
+    code=$?
+fi
 
 # Deactivate the Python virtualenv in case the user "source"'d this script
 log "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment."

toolchain/bench.yaml

@@ -1,3 +1,9 @@
-- name: 1D_bubblescreen
-  path: examples/1D_bubblescreen/case.py
+- name: 3D_shockdroplet
+  path: examples/3D_shockdroplet/case.py
+  args: []
+- name: 3D_sphbubcollapse
+  path: examples/3D_sphbubcollapse/case.py
+  args: []
+- name: 3D_turb_mixing
+  path: examples/3D_turb_mixing/case.py
   args: []

toolchain/mfc.py

@@ -12,11 +12,11 @@
 
 def __print_greeting():
     MFC_LOGO_LINES       = MFC_LOGO.splitlines()
-    max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ])
+    max_logo_line_length = max(len(line) for line in MFC_LOGO_LINES)
 
     host_line    = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]"
     targets_line = f"[bold]--targets {format_list_to_string(ARG('targets'), 'magenta', 'None')}[/bold]"
-    help_line    = "$ ./mfc.sh \[build, run, test, clean, count, packer] --help"
+    help_line    = "$ ./mfc.sh [build, run, test, clean, count, packer] --help"
 
     MFC_SIDEBAR_LINES = [
         "",
@@ -47,7 +47,7 @@ def __checks():
         raise MFCException("CMake is required to build MFC but couldn't be located on your system. Please ensure it installed and discoverable (e.g in your system's $PATH).")
 
 
-def __run():    
+def __run():
     {"test":   test.test,   "run":   run.run,    "build": build.build,
      "clean":  build.clean, "bench": bench.bench, "count": count.count,
      "packer": packer.packer
@@ -60,7 +60,7 @@ def __run():
         state.gARG = args.parse(state.gCFG)
 
         lock.switch(state.MFCConfig.from_dict(state.gARG))
-        
+
         __print_greeting()
         __checks()
         __run()

toolchain/mfc/args.py

@@ -3,36 +3,35 @@
 from .build     import TARGETS, DEFAULT_TARGETS, DEPENDENCY_TARGETS
 from .common    import MFCException, format_list_to_string
 from .test.test import CASES as TEST_CASES
-from .packer    import packer
 
 def parse(config):
     from .run.engines  import ENGINES
     from .run.mpi_bins import BINARIES
 
     parser = argparse.ArgumentParser(
         prog="./mfc.sh",
-        description=f"""\
+        description="""\
 Welcome to the MFC master script. This tool automates and manages building, testing, \
 running, and cleaning of MFC in various configurations on all supported platforms. \
 The README documents this tool and its various commands in more detail. To get \
 started, run ./mfc.sh build -h.""",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parsers = parser.add_subparsers(dest="command")    
+    parsers = parser.add_subparsers(dest="command")
     run     = parsers.add_parser(name="run",    help="Run a case with MFC.",            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     test    = parsers.add_parser(name="test",   help="Run MFC's test suite.",           formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     build   = parsers.add_parser(name="build",  help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     clean   = parsers.add_parser(name="clean",  help="Clean build artifacts.",          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     bench   = parsers.add_parser(name="bench",  help="Benchmark MFC (for CI).",         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     count   = parsers.add_parser(name="count",  help="Count LOC in MFC.",               formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     packer  = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-        
+
     packers = packer.add_subparsers(dest="packer")
     pack = packers.add_parser(name="pack", help="Pack a case into a single file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     pack.add_argument("input", metavar="INPUT", type=str, default="", help="Input file of case to pack.")
     pack.add_argument("-o", "--output", metavar="OUTPUT", type=str, default=None, help="Base name of output file.")
-    
+
     compare = packers.add_parser(name="compare", help="Compare two cases.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     compare.add_argument("input1", metavar="INPUT1", type=str, default=None, help="First pack file.")
     compare.add_argument("input2", metavar="INPUT2", type=str, default=None, help="Second pack file.")
@@ -54,7 +53,7 @@ def add_common_arguments(p, mask = None):
                 p.add_argument(f"--no-{f.name}", action="store_false", dest=f.name, help=f"Turn the {f.name} option OFF.")
 
             p.set_defaults(**{ f.name: getattr(config, f.name) for f in dataclasses.fields(config) })
-            
+
         if "j" not in mask:
             p.add_argument("-j", "--jobs", metavar="JOBS", type=int, default=1, help="Allows for JOBS concurrent jobs.")
 
@@ -91,7 +90,7 @@ def add_common_arguments(p, mask = None):
     test.add_argument("-m", "--max-attempts", type=int, default=3, help="Maximum number of attempts to run a test.")
 
     test.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
-    
+
     test_meg = test.add_mutually_exclusive_group()
     test_meg.add_argument("--generate",          action="store_true", default=False, help="(Test Generation) Generate golden files.")
     test_meg.add_argument("--add-new-variables", action="store_true", default=False, help="(Test Generation) If new variables are found in D/ when running tests, add them to the golden files.")
@@ -154,7 +153,7 @@ def add_common_arguments(p, mask = None):
     # build's --case-optimization and --input depend on each other
     if args["command"] == "build":
         if (args["input"] is not None) ^ args["case_optimization"] :
-            raise MFCException(f"./mfc.sh build's --case-optimization requires --input")
+            raise MFCException("./mfc.sh build's --case-optimization requires --input")
 
     # Input files to absolute paths
     for e in ["input", "input1", "input2"]:

toolchain/mfc/bench.py

@@ -2,9 +2,8 @@
 
 from .printer import cons
 from .state   import ARG, CFG
-from .build   import PRE_PROCESS, SIMULATION, build_targets
+from .build   import PRE_PROCESS, SIMULATION
 from .common  import system, MFC_BENCH_FILEPATH, file_load_yaml, file_dump_yaml
-from .        import sched
 
 
 @dataclasses.dataclass

toolchain/mfc/build.py

@@ -1,8 +1,7 @@
-import re, os, typing, hashlib, dataclasses
+import os, typing, hashlib, dataclasses
 
 from .common    import MFCException, system, delete_directory, create_directory
 from .state     import ARG, CFG
-from .printer   import cons
 from .run       import input
 
 @dataclasses.dataclass
@@ -26,10 +25,10 @@ def compute(self) -> typing.Set:
     isRequired:   bool             # Should it always be built? (no matter what -t | --targets is)
     requires:     Dependencies     # Build dependencies of the target
     runOrder:     int              # For MFC Targets: Order in which targets should logically run
-    
+
     def __hash__(self) -> int:
         return hash(self.name)
-    
+
     # Get path to directory that will store the build files
     def get_build_dirpath(self) -> str:
         subdir = 'dependencies' if self.isDependency else CFG().make_slug()
@@ -66,7 +65,7 @@ def get_install_dirpath(self) -> str:
             "install",
             'dependencies' if self.isDependency else CFG().make_slug()
         ])
-    
+
     def get_install_binpath(self) -> str:
         # <root>/install/<slug>/bin/<target>
         return os.sep.join([self.get_install_dirpath(), "bin", self.name])
@@ -104,17 +103,17 @@ def configure(self):
         build_dirpath   = self.get_build_dirpath()
         cmake_dirpath   = self.get_cmake_dirpath()
         install_dirpath = self.get_install_dirpath()
-        
+
         install_prefixes = ';'.join([install_dirpath, get_dependency_install_dirpath()])
 
         flags: list = self.flags.copy() + [
             # Disable CMake warnings intended for developers (us).
             # See: https://cmake.org/cmake/help/latest/manual/cmake.1.html.
-            f"-Wno-dev",
+            "-Wno-dev",
             # Save a compile_commands.json file with the compile commands used to
             # build the configured targets. This is mostly useful for debugging.
             # See: https://cmake.org/cmake/help/latest/variable/CMAKE_EXPORT_COMPILE_COMMANDS.html.
-            f"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
+            "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
             # Set build type (e.g Debug, Release, etc.).
             # See: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
             f"-DCMAKE_BUILD_TYPE={'Debug' if ARG('debug') else 'Release'}",
@@ -160,7 +159,7 @@ def install(self):
         install = ["cmake", "--install", self.get_build_dirpath()]
 
         system(install, exception_text=f"Failed to install the [bold magenta]{self.name}[/bold magenta] target.")
-        
+
     def clean(self):
         build_dirpath = self.get_build_dirpath()
 
@@ -196,7 +195,7 @@ def clean(self):
 def get_target(target: typing.Union[str, MFCTarget]) -> MFCTarget:
     if isinstance(target, MFCTarget):
         return target
-    
+
     if target in TARGET_MAP:
         return TARGET_MAP[target]
 
@@ -218,7 +217,7 @@ def build_target(target: typing.Union[MFCTarget, str], history: typing.Set[str]
         history = set()
 
     t = get_target(target)
-    
+
     if t.name in history or not t.is_buildable():
         return
 
@@ -235,7 +234,7 @@ def build_target(target: typing.Union[MFCTarget, str], history: typing.Set[str]
 def build_targets(targets: typing.Iterable[typing.Union[MFCTarget, str]], history: typing.Set[str] = None):
     if history is None:
         history = set()
- 
+
     for target in list(REQUIRED_TARGETS) + targets:
         build_target(target, history)
 
@@ -245,7 +244,7 @@ def clean_target(target: typing.Union[MFCTarget, str], history: typing.Set[str]
         history = set()
 
     t = get_target(target)
-    
+
     if t.name in history or not t.is_buildable():
         return
 

toolchain/mfc/case.py

@@ -6,7 +6,7 @@ class Case:
 
     def __init__(self, params: dict) -> None:
         self.params = copy.deepcopy(params)
-    
+
     def get_parameters(self) -> str:
         return self.params.keys()
 

toolchain/mfc/common.py

@@ -1,6 +1,4 @@
-import os, yaml, typing, shutil, subprocess, dataclasses
-
-from datetime import datetime
+import os, yaml, typing, shutil, subprocess
 
 from .printer import cons
 
@@ -13,7 +11,7 @@
 MFC_LOCK_FILEPATH  = abspath(f"{MFC_SUBDIR}/lock.yaml")
 MFC_BENCH_FILEPATH = abspath(f"{MFC_ROOTDIR}/toolchain/bench.yaml")
 
-MFC_LOGO = f"""
+MFC_LOGO = """
      .=++*:          -+*+=.
     :+   -*-        ==   =* .
   :*+      ==      ++    .+-
@@ -40,7 +38,7 @@ def system(command: typing.List[str], no_exception: bool = False, exception_text
         cons.print(no_indent=True)
 
     cons.print(f"$ {' '.join(cmd)}")
-    
+
     if stdout != subprocess.DEVNULL:
         cons.print(no_indent=True)
 
@@ -121,7 +119,7 @@ def get_program_output(arguments: typing.List[str] = None, cwd=None):
     return (proc.communicate()[0].decode(), proc.returncode)
 
 
-def get_py_program_output(filepath: str, arguments: typing.List[str] = None):    
+def get_py_program_output(filepath: str, arguments: typing.List[str] = None):
     dirpath  = os.path.abspath (os.path.dirname(filepath))
     filename = os.path.basename(filepath)
 
@@ -156,12 +154,12 @@ def does_command_exist(s: str) -> bool:
 def format_list_to_string(arr: list, item_style=None, empty=None):
     if empty is None:
         empty = "nothing"
-    
+
     pre, post = "", ""
     if item_style is not None:
         pre  = f"[{item_style}]"
         post = f"[/{item_style}]"
-    
+
     if len(arr) == 0:
         return f"{pre}{empty}{post}"
 
@@ -189,6 +187,7 @@ def find(predicate, arr: list):
     return None, None
 
 
+# pylint: disable=redefined-builtin
 def quit(sig):
     os.kill(os.getpid(), sig)
 
@@ -212,10 +211,10 @@ def get_loaded_modules() -> typing.List[str]:
 def is_number(x: str) -> bool:
     if x is None:
         return False
-    
+
     if isinstance(x, int) or isinstance(x, float):
         return True
-    
+
     try:
         float(x)
         return True

toolchain/mfc/count.py

@@ -25,17 +25,17 @@ def count():
 
     cons.print(f"[bold]Counting lines of code in {target_str_list}[/bold] (excluding whitespace lines)")
     cons.indent()
-  
+
     total = 0
     for codedir in ['common'] + ARG("targets"):
-        dirfiles, dircount = handle_dir(os.path.join(MFC_ROOTDIR, 'src', codedir))        
+        dirfiles, dircount = handle_dir(os.path.join(MFC_ROOTDIR, 'src', codedir))
         table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE)
         table.add_column(f"File (in [magenta]{codedir}[/magenta])", justify="left")
         table.add_column(f"Lines ([cyan]{dircount}[/cyan])", justify="right")
-        
+
         for filepath, count in dirfiles:
             table.add_row(f"{os.path.basename(filepath)}", f"[bold cyan]{count}[/bold cyan]")
-        
+
         total += dircount
 
         cons.raw.print(table)