Merge pull request #32 from aurora-multiphysics/sapphire

Sapphire Rapids and other build updates

Makefile

@@ -28,23 +28,27 @@ ALL_MODULES                 := no
 
 CHEMICAL_REACTIONS          := no
 CONTACT                     := yes
+ELECTROMAGNETICS            := no
 EXTERNAL_PETSC_SOLVER       := no
 FLUID_PROPERTIES            := yes
 FSI                         := no
 FUNCTIONAL_EXPANSION_TOOLS  := no
 GEOCHEMISTRY                := no
-HEAT_CONDUCTION             := yes
+HEAT_TRANSFER               := yes
 LEVEL_SET                   := yes
 MISC                        := no
 NAVIER_STOKES               := yes
+OPTIMIZATION                := no
+PERIDYNAMICS                := no
 PHASE_FIELD                 := no
 POROUS_FLOW                 := no
 RAY_TRACING                 := no
 RDG                         := no
 REACTOR                     := yes
 RICHARDS                    := no
+SOLID_MECHANICS             := yes
 STOCHASTIC_TOOLS            := no
-TENSOR_MECHANICS            := yes
+THERMAL_HYDRAULICS          := no
 XFEM                        := no
 
 include $(MOOSE_DIR)/modules/modules.mk

examples/fluids/duct3D_ke.i

@@ -185,8 +185,8 @@
 [Executioner]
   type = Steady
   solve_type = NEWTON
-  petsc_options_iname = '-pc_type'
-  petsc_options_value = 'ilu'
+  petsc_options_iname = '-pc_type -pc_factor_shift_type'
+  petsc_options_value = 'ilu      NONZERO'
   l_max_its = 30
 []
 

examples/fluids/duct3D_laminar_AD.i

@@ -13,8 +13,10 @@
   [velocity]
     order = FIRST
     family = LAGRANGE_VEC
+    scaling = 20
   []
   [p]
+    scaling = 0.3
   []
 []
 
@@ -106,7 +108,6 @@
   petsc_options_iname = '-pc_type'
   petsc_options_value = 'ilu'
   l_max_its = 30
-  automatic_scaling = true
 []
 
 [Outputs]

scripts/NSFVBase.patch

@@ -0,0 +1,22 @@
+diff --git a/modules/navier_stokes/include/base/NSFVBase.h b/modules/navier_stokes/include/base/NSFVBase.h
+index e90b4a85..7a9ac856 100644
+--- a/modules/navier_stokes/include/base/NSFVBase.h
++++ b/modules/navier_stokes/include/base/NSFVBase.h
+@@ -3537,12 +3537,13 @@ NSFVBase<BaseType>::checkBlockwiseConsistency(const std::string block_param_name
+                        "Block '" + block +
+                            "' is not present in the block restriction of the fluid flow action!");
+
+-    for (const auto & param_name : parameter_names)
++    for (unsigned int param_i = 0; param_i < parameter_names.size(); ++param_i)
+     {
+-      const std::vector<T> & param_vector = parameters().template get<std::vector<T>>(param_name);
++      const std::vector<T> & param_vector =
++          parameters().template get<std::vector<T>>(parameter_names[param_i]);
+       if (block_names.size() != param_vector.size())
+-        paramError(param_name,
+-                   "The number of entries in '" + param_name + "' (" +
++        paramError(parameter_names[param_i],
++                   "The number of entries in '" + parameter_names[param_i] + "' (" +
+                        std::to_string(param_vector.size()) +
+                        ") is not the same as the number of blocks"
+                        " (" +

scripts/csd3_python_prep.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# This scripts installs python pre-requisites for Proteus on CSD3.
+# It should be run once before the install script.
+
+module load python/3.11.0-icl
+pip3.11 install --user --upgrade packaging
+pip3.11 install --user --upgrade pyyaml

scripts/install_fedora.sh → scripts/install_csd3_sapphire.sh

@@ -1,60 +1,47 @@
 #!/bin/bash
 
-# This script installs Proteus on Fedora,
+# This script installs Proteus on CSD3,
 # including a MOOSE framework build in the $HOME directory.
-# Optimised to the native system architecture.
+# Optimised to the native system architecture on the sapphire partition.
 # A .proteus_profile script is added to the $HOME directory.
 # This script is intended to be used from the proteus directory
-#   ./scripts/install_fedora.sh
+#   ./scripts/install_csd3_sapphire.sh
 # Use the installation by typing:
 #   source $HOME/.proteus_profile
 
 export PROTEUS_DIR=`pwd`
 
-# If MOOSE_JOBS is unset, set to 1
-if [ -z $MOOSE_JOBS ]; then
-  export MOOSE_JOBS=1
-fi
-
-# Install pre-requisites
-
-sudo dnf install gcc gcc-c++ gcc-fortran cmake bison flex git
-sudo dnf install python3 python3-devel
-sudo dnf install openmpi openmpi-devel
-sudo dnf install libtirpc-devel zlib-devel
-sudo dnf install perl-File-Compare
-
 # Make Proteus profile
 
-echo "export CC=mpicc" > $HOME/.proteus_profile
+echo "module purge" > $HOME/.proteus_profile
+echo "module load rhel8/default-sar python/3.11.0-icl" >> $HOME/.proteus_profile
+echo "export CC=mpicc" >> $HOME/.proteus_profile
 echo "export CXX=mpicxx" >> $HOME/.proteus_profile
 echo "export F90=mpif90" >> $HOME/.proteus_profile
 echo "export F77=mpif77" >> $HOME/.proteus_profile
 echo "export FC=mpif90" >> $HOME/.proteus_profile
 echo "export MOOSE_DIR="$HOME"/moose" >> $HOME/.proteus_profile
 echo "export PATH=\$PATH:"$PROTEUS_DIR >> $HOME/.proteus_profile
-echo "module load mpi/openmpi" >> $HOME/.proteus_profile
+echo "unset I_MPI_PMI_LIBRARY" >> $HOME/.proteus_profile
 source $HOME/.proteus_profile
 
 # Clone MOOSE from git
 
 cd $HOME
-git clone https://github.com/idaholab/moose.git
+git clone --depth=1 https://github.com/idaholab/moose.git
+
+# Set MOOSE jobs to 4 (max allowed on login node)
+export MOOSE_JOBS=4 METHODS="opt"
 
 # Build PETSc
 
 cd $MOOSE_DIR
 unset PETSC_DIR PETSC_ARCH
-./scripts/update_and_rebuild_petsc.sh \
-CC=$CC CXX=$CXX F90=$F90 F77=$F77 FC=$FC \
---CXXOPTFLAGS="-O3 -march=native" \
---COPTFLAGS="-O3 -march=native" \
---FOPTFLAGS="-O3 -march=native" \
---download-mumps=0 --download-superlu_dist=0
+./scripts/update_and_rebuild_petsc.sh --download-cmake
 
 # Build libMesh
 
-METHODS="opt" ./scripts/update_and_rebuild_libmesh.sh --with-mpi
+./scripts/update_and_rebuild_libmesh.sh --with-mpi
 
 # Build WASP
 
@@ -67,6 +54,10 @@ METHODS="opt" ./scripts/update_and_rebuild_libmesh.sh --with-mpi
 
 ./configure --with-derivative-size=81
 
+# Apply NSFV patch
+
+git apply $PROTEUS_DIR/scripts/NSFVBase.patch
+
 cd $PROTEUS_DIR
 make -j $MOOSE_JOBS
 

scripts/install_ubuntu.sh

@@ -15,6 +15,7 @@ export PROTEUS_DIR=`pwd`
 if [ -z $MOOSE_JOBS ]; then
   export MOOSE_JOBS=1
 fi
+export METHODS="opt"
 
 # Install pre-requisites
 
@@ -43,15 +44,13 @@ git clone https://github.com/idaholab/moose.git
 cd $MOOSE_DIR
 unset PETSC_DIR PETSC_ARCH
 ./scripts/update_and_rebuild_petsc.sh \
-CC=$CC CXX=$CXX F90=$F90 F77=$F77 FC=$FC \
 --CXXOPTFLAGS="-O3 -march=native" \
 --COPTFLAGS="-O3 -march=native" \
---FOPTFLAGS="-O3 -march=native" \
---download-mumps=0 --download-superlu_dist=0
+--FOPTFLAGS="-O3 -march=native"
 
 # Build libMesh
 
-METHODS="opt" ./scripts/update_and_rebuild_libmesh.sh --with-mpi
+./scripts/update_and_rebuild_libmesh.sh --with-mpi
 
 # Build WASP
 

scripts/slurm_submit.cclake

@@ -0,0 +1,20 @@
+#!/bin/bash
+#SBATCH -J MOOSE
+#SBATCH -A <project>
+#SBATCH -p cclake
+#SBATCH --nodes=2
+#! 56 cores per node 3420 MiB per core
+#SBATCH --ntasks=112
+#SBATCH --time=36:00:00
+#SBATCH --mail-type=NONE
+
+cd $SLURM_SUBMIT_DIR
+
+echo -e "Changed directory to `pwd`.\n"
+echo -e "JobID: $SLURM_JOB_ID\n======"
+echo "Time: `date`"
+echo "Running on master node: `hostname`"
+echo "Current directory: `pwd`"
+
+source ~/.proteus_profile
+mpirun -np 112 proteus-opt --distributed-mesh -t -r 1 -i ~/proteus/examples/benchmark/diffusion_3D_32.i &> log.benchmark

scripts/slurm_submit.sapphire

@@ -0,0 +1,20 @@
+#!/bin/bash
+#SBATCH -J MOOSE
+#SBATCH -A <project>
+#SBATCH -p sapphire
+#SBATCH --nodes=2
+#! 112 cores per node
+#SBATCH --ntasks=224
+#SBATCH --time=36:00:00
+#SBATCH --mail-type=NONE
+
+cd $SLURM_SUBMIT_DIR
+
+echo -e "Changed directory to `pwd`.\n"
+echo -e "JobID: $SLURM_JOB_ID\n======"
+echo "Time: `date`"
+echo "Running on master node: `hostname`"
+echo "Current directory: `pwd`"
+
+source ~/.proteus_profile
+mpirun -np 224 proteus-opt --distributed-mesh -t -r 1 -i ~/proteus/examples/benchmark/diffusion_3D_32.i &> log.benchmark

src/base/ProteusApp.C

@@ -8,10 +8,7 @@ InputParameters
 ProteusApp::validParams()
 {
   InputParameters params = MooseApp::validParams();
-
-  // Do not use legacy material output, i.e., output properties on INITIAL as well as TIMESTEP_END
   params.set<bool>("use_legacy_material_output") = false;
-
   return params;
 }
 

src/main.C

@@ -1,36 +1,10 @@
-//* This file is part of the MOOSE framework
-//* https://www.mooseframework.org
-//*
-//* All rights reserved, see COPYRIGHT for full restrictions
-//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
-//*
-//* Licensed under LGPL 2.1, please see LICENSE for details
-//* https://www.gnu.org/licenses/lgpl-2.1.html
-
 #include "ProteusTestApp.h"
-#include "MooseInit.h"
-#include "Moose.h"
-#include "MooseApp.h"
-#include "AppFactory.h"
-
-// Create a performance log
-PerfLog Moose::perf_log("Proteus");
+#include "MooseMain.h"
 
-// Begin the main program.
 int
 main(int argc, char * argv[])
 {
-  // Initialize MPI, solvers and MOOSE
-  MooseInit init(argc, argv);
-
-  // Register this application's MooseApp and any it depends on
-  ProteusTestApp::registerApps();
-
-  // Create an instance of the application and store it in a smart pointer for easy cleanup
-  std::shared_ptr<MooseApp> app = AppFactory::createAppShared("ProteusTestApp", argc, argv);
-
-  // Execute the application
-  app->run();
+  Moose::main<ProteusTestApp>(argc, argv);
 
   return 0;
 }

src/materials/IRMINSADMaterial.C

@@ -119,7 +119,7 @@ IRMINSADMaterial::subdomainSetup()
     addMooseVariableDependency(&disp_x);
     _disp_x_dot = &disp_x.adUDot();
     _disp_x_sys_num = disp_x.sys().number();
-    _disp_x_num = (disp_x.kind() == Moose::VarKindType::VAR_NONLINEAR) &&
+    _disp_x_num = (disp_x.kind() == Moose::VarKindType::VAR_SOLVER) &&
                           (_disp_x_sys_num == _fe_problem.currentNonlinearSystem().number())
                       ? disp_x.number()
                       : libMesh::invalid_uint;
@@ -131,7 +131,7 @@ IRMINSADMaterial::subdomainSetup()
       _disp_y_dot = &disp_y.adUDot();
       _disp_y_sys_num = disp_y.sys().number();
       _disp_y_num =
-          disp_y.kind() == (Moose::VarKindType::VAR_NONLINEAR &&
+          disp_y.kind() == (Moose::VarKindType::VAR_SOLVER &&
                             (_disp_y_sys_num == _fe_problem.currentNonlinearSystem().number()))
               ? disp_y.number()
               : libMesh::invalid_uint;
@@ -150,7 +150,7 @@ IRMINSADMaterial::subdomainSetup()
       _disp_z_dot = &disp_z.adUDot();
       _disp_z_sys_num = disp_z.sys().number();
       _disp_z_num =
-          disp_z.kind() == (Moose::VarKindType::VAR_NONLINEAR &&
+          disp_z.kind() == (Moose::VarKindType::VAR_SOLVER &&
                             (_disp_z_sys_num == _fe_problem.currentNonlinearSystem().number()))
               ? disp_z.number()
               : libMesh::invalid_uint;