finish and put back other ci

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker Publisher
+
+on:
+  push:
+    branches: [ "master" ]
+    paths:
+      - toolchain/Dockerfile
+
+  workflow_dispatch:
+
+jobs:
+  push_to_registry:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Build & Publish thereto
+      uses: docker/build-push-action@v4
+      with:
+        file: toolchain/Dockerfile
+        push: true
+        tags: ${{ secrets.DOCKER_USERNAME }}/mfc:latest
+

.github/workflows/docs.yml

@@ -0,0 +1,54 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - master
+
+  workflow_dispatch:
+
+jobs:
+  docs:
+    name:    Build & Publish
+    runs-on: ubuntu-latest
+
+    if: github.repository == 'MFlowCode/MFC'
+    concurrency:
+      group: docs-publish
+      cancel-in-progress: true
+
+    steps:
+    - uses: actions/checkout@v3
+
+    # We build doxygen from source because of
+    # https://github.com/doxygen/doxygen/issues/9016
+    - name: Build Doxygen
+      run:  |
+        sudo apt install cmake ninja-build graphviz graphviz
+        git clone https://github.com/doxygen/doxygen.git ../doxygen
+        cmake -S ../doxygen -B ../doxygen/build -G Ninja
+        sudo ninja -C ../doxygen/build install
+
+    - name: Build Documentation
+      run:  |
+        pip3 install fypp
+        cmake -S . -B build -G Ninja --install-prefix=$(pwd)/build/install -D MFC_DOCUMENTATION=ON
+        ninja -C build install
+
+    - name: Publish Documentation
+      run:  |
+        set +e
+        git ls-remote "${{ secrets.DOC_PUSH_URL }}" -q
+        if [ "$?" -ne "0" ]; then exit 0; fi
+        set -e
+        git config --global user.name  'MFC Action'
+        git config --global user.email '<>'
+        git clone "${{ secrets.DOC_PUSH_URL }}" ../www
+        rm -rf ../www/*
+        mv build/install/docs/mfc/* ../www/
+        git -C ../www add -A
+        git -C ../www commit -m "Docs @ ${GITHUB_SHA::7}" || true
+        git -C ../www push
+
+# DOC_PUSH_URL should be of the format:
+# --> https://<username>:<token>@github.com/<username>/<repository>

.github/workflows/lint.yml

@@ -0,0 +1,19 @@
+name: Lint
+
+on:
+  push:
+
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  docs:
+    name:    Lint Toolchain
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Lint the toolchain
+      run: ./mfc.sh lint

.github/workflows/phoenix/submit.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+sbatch_cpu_opts="\
+#SBATCH -p cpu-small               # partition
+#SBATCH --ntasks-per-node=24       # Number of cores per node required
+#SBATCH --mem-per-cpu=2G           # Memory per core\
+"
+
+sbatch_gpu_opts="\
+#SBATCH -CV100-16GB
+#SBATCH -G2\
+"
+
+if [ "$2" == "cpu" ]; then
+    sbatch_device_opts="$sbatch_cpu_opts"
+elif [ "$2" == "gpu" ]; then
+    sbatch_device_opts="$sbatch_gpu_opts"
+else
+    usage
+    exit 1
+fi
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -Jshb-$job_slug            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1                        # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 04:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+
+. ./mfc.sh load -c p -m $2
+
+$sbatch_script_contents
+
+EOT
+

.github/workflows/phoenix/test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+build_opts=""
+if [ "$job_device" == "gpu" ]; then
+    build_opts="--gpu"
+fi
+
+./mfc.sh build -j 8 $build_opts
+
+n_test_threads=8
+
+if [ "$job_device" == "gpu" ]; then
+    gpu_count=$(nvidia-smi -L | wc -l)        # number of GPUs on node
+    gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
+    device_opts="-g $gpu_ids"
+    n_test_threads=`expr $gpu_count \* 2`
+fi
+
+./mfc.sh test -a -j $n_test_threads $device_opts -- -c phoenix
+

.github/workflows/pretty.yml

@@ -0,0 +1,21 @@
+name: Pretty
+
+on:
+  push:
+
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  docs:
+    name:    Code formatting
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Check formatting
+      run: |
+        ./mfc.sh format
+        git diff --exit-code

.github/workflows/spelling.yml

@@ -0,0 +1,15 @@
+name: Spell Check
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  run:
+    name: Spell Check
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Spell Check
+      uses: crate-ci/typos@master

.github/workflows/test.yml

@@ -0,0 +1,126 @@
+name: 'Test Suite'
+
+on:
+  push:
+    paths:
+      - '**.f90'
+      - '**.fpp'
+      - '**.py'
+      - '**.yml'
+      - 'mfc.sh'
+      - 'golden.txt'
+      - 'CMakeLists.txt'
+      - 'requirements.txt'
+
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  github:
+    name: Github
+    strategy:
+      matrix:
+        os:    ['ubuntu', 'macos']
+        mpi:   ['mpi', 'no-mpi']
+        debug: ['debug', 'no-debug']
+        intel: [true, false]
+        exclude:
+          - intel: true
+            os:    macos
+      fail-fast: false
+    continue-on-error: true
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v3
+
+      - name: Setup MacOS
+        if:   matrix.os == 'macos'
+        run: |
+          echo "CC=gcc-13"      >> $GITHUB_ENV
+          echo "CXX=g++-13"     >> $GITHUB_ENV
+          echo "FC=gfortran-13" >> $GITHUB_ENV
+          brew install wget make python make cmake coreutils gcc@13
+          
+      - name: (MacOS) Build OpenMPI
+        if:   matrix.os == 'macos' && matrix.mpi == 'mpi'
+        run: |
+          echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV
+          echo "OMPI_CXX=g++-13"     >> $GITHUB_ENV
+          echo "OMPI_MPICC=gcc-13"   >> $GITHUB_ENV
+          HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-13 --verbose --build-from-source open-mpi
+
+      - name: Setup Ubuntu
+        if:   matrix.os == 'ubuntu' && matrix.intel == false
+        run: |
+           sudo apt update -y
+           sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
+     
+      - name: Setup Ubuntu (Intel)
+        if:   matrix.os == 'ubuntu' && matrix.intel == true
+        run: |
+          sudo apt install tar wget make cmake python3 python3-dev
+          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+          sudo apt-get update
+          sudo apt-get install -y intel-oneapi-common-vars intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-mkl-2021.4.0 intel-oneapi-mpi-2021.7.1 intel-oneapi-mpi-devel-2021.7.1
+          source /opt/intel/oneapi/setvars.sh
+          echo "CXX=$(which icpc)"          >> $GITHUB_ENV
+          echo "CC=$(which icc)"            >> $GITHUB_ENV
+          echo "FC=$(which ifort)"          >> $GITHUB_ENV
+          echo "OMPI_FC=$(which ifort)"     >> $GITHUB_ENV
+          echo "OMPI_CXX=$(which icpc)"     >> $GITHUB_ENV
+          echo "OMPI_MPICC=$(which icc)"    >> $GITHUB_ENV
+          echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
+          echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
+
+      - name: Build
+        run:  |
+          if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
+          /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} 
+
+      - name: Test
+        run:  |
+          if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
+          /bin/bash mfc.sh test -j $(nproc) $OPT1 $OPT2
+        env:
+          OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
+          OPT2: ${{ matrix.debug == 'debug' && '-% 20' || '' }}
+
+  docker:
+    name:    Github | Docker
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone
+      uses: actions/checkout@v3
+
+    - name: Test
+      run:  sudo ./mfc.sh docker ./mfc.sh test -j $(nproc) -a
+
+  self:
+    name: Georgia Tech | Phoenix (NVHPC)
+    if: github.repository == 'MFlowCode/MFC'
+    continue-on-error: true
+    strategy:
+      matrix:
+        device: ['cpu', 'gpu']
+    runs-on:
+      group:  phoenix
+      labels: self-hosted
+    steps:
+      - name: Clone
+        uses: actions/checkout@v3
+
+      - name: Build & Test
+        run:  bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
+
+      - name: Archive Logs
+        uses: actions/upload-artifact@v3
+        if:   always()
+        with:
+          name: logs
+          path: test-${{ matrix.device }}.out
+