Skip to content

Commit

Permalink
Merge pull request #161 from anandrdbz/patch-1
Browse files Browse the repository at this point in the history
  • Loading branch information
@sbryngelson
sbryngelson authored Aug 1, 2023
2 parents c0e510e + be8dc4a commit 363d584
Show file tree
Hide file tree
Showing 179 changed files with 683 additions and 254 deletions.
36 changes: 27 additions & 9 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ on:
pull_request:

workflow_dispatch:

jobs:
github:
name: Github
Expand All @@ -24,24 +24,25 @@ jobs:
os: ['ubuntu', 'macos']
mpi: ['mpi', 'no-mpi']
debug: ['debug', 'no-debug']
intel: [false]
intel: [true, false]
fail-fast: false
continue-on-error: true
runs-on: ${{ matrix.os }}-latest
steps:
- name: Clone
if: matrix.os == 'ubuntu' || (matrix.os == 'macos' && matrix.intel == false)
uses: actions/checkout@v3

- name: Setup MacOS
if: matrix.os == 'macos'
if: matrix.os == 'macos' && matrix.intel == false
run: |
echo "CC=gcc-13" >> $GITHUB_ENV
echo "CXX=g++-13" >> $GITHUB_ENV
echo "FC=gfortran-13" >> $GITHUB_ENV
brew install wget make python make cmake coreutils gcc@13
- name: (MacOS) Build OpenMPI
if: matrix.os == 'macos' && matrix.mpi == 'mpi'
if: matrix.os == 'macos' && matrix.mpi == 'mpi' && matrix.intel == false
run: |
echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV
echo "OMPI_CXX=g++-13" >> $GITHUB_ENV
Expand All @@ -51,9 +52,9 @@ jobs:
- name: Setup Ubuntu
if: matrix.os == 'ubuntu' && matrix.intel == false
run: |
sudo apt update -y
sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
sudo apt update -y
sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
- name: Setup Ubuntu (Intel)
if: matrix.os == 'ubuntu' && matrix.intel == true
run: |
Expand All @@ -72,12 +73,29 @@ jobs:
echo "OMPI_CXX=$(which icpc)" >> $GITHUB_ENV
echo "OMPI_MPICC=$(which icc)" >> $GITHUB_ENV
echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
- name: Build-intel
if: matrix.intel == true && matrix.os == 'ubuntu'
run: |
source /opt/intel/oneapi/setvars.sh
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
- name: Build
run: /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
if: matrix.intel == false
run: |
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
- name: Test-intel
if: matrix.intel == true && matrix.os == 'ubuntu'
run: |
source /opt/intel/oneapi/setvars.sh
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
- name: Test
run: /bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
if: matrix.intel == false
run: |
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
docker:
name: Github | Docker
Expand Down
1 change: 0 additions & 1 deletion src/common/m_variables_conversion.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,6 @@ contains
s_convert_to_mixture_variables => &
s_convert_species_to_mixture_variables
end if

end subroutine s_initialize_variables_conversion_module ! --------------

!Initialize mv at the quadrature nodes based on the initialized moments and sigma
Expand Down
84 changes: 42 additions & 42 deletions src/post_process/m_data_input.f90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,15 +405,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'x')

do i = 1, offset_x%beg
x_cb(-1 - i) = x_cb(-i) - dx(-i)
end do
end if

do i = 1, buff_size
x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
end do
do i = 1, offset_x%beg
x_cb(-1 - i) = x_cb(-i) - dx(-i)
end do

end if
do i = 1, buff_size
x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
end do

! Ghost-cell extrapolation BC at the end
if (bc_x%end <= -3) then
Expand Down Expand Up @@ -441,15 +441,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'x')

do i = 1, offset_x%end
x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
end do
end if

do i = 1, buff_size
x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
end do
do i = 1, offset_x%end
x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
end do

end if
do i = 1, buff_size
x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
end do

! END: Populating Buffer Regions in the x-direction ================

Expand Down Expand Up @@ -483,15 +483,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'y')

do i = 1, offset_y%beg
y_cb(-1 - i) = y_cb(-i) - dy(-i)
end do
end if

do i = 1, buff_size
y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
end do
do i = 1, offset_y%beg
y_cb(-1 - i) = y_cb(-i) - dy(-i)
end do

end if
do i = 1, buff_size
y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
end do

! Ghost-cell extrapolation BC at the end
if (bc_y%end <= -3) then
Expand Down Expand Up @@ -519,15 +519,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'y')

do i = 1, offset_y%end
y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
end do
end if

do i = 1, buff_size
y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
end do
do i = 1, offset_y%end
y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
end do

end if
do i = 1, buff_size
y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
end do

! END: Populating Buffer Regions in the y-direction ================

Expand Down Expand Up @@ -561,15 +561,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'z')

do i = 1, offset_z%beg
z_cb(-1 - i) = z_cb(-i) - dz(-i)
end do
end if

do i = 1, buff_size
z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
end do
do i = 1, offset_z%beg
z_cb(-1 - i) = z_cb(-i) - dz(-i)
end do

end if
do i = 1, buff_size
z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
end do

! Ghost-cell extrapolation BC at the end
if (bc_z%end <= -3) then
Expand Down Expand Up @@ -597,15 +597,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------

call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'z')

do i = 1, offset_z%end
z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
end do
end if

do i = 1, buff_size
z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
end do
do i = 1, offset_z%end
z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
end do

end if
do i = 1, buff_size
z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
end do

end if

Expand Down
52 changes: 45 additions & 7 deletions src/post_process/m_mpi_proxy.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ module m_mpi_proxy
use m_global_parameters !< Global parameters for the code
use m_mpi_common
use ieee_arithmetic
! ==========================================================================
implicit none
Expand Down Expand Up @@ -516,8 +518,8 @@ contains
! Boundary condition at the beginning
if (proc_coords(2) > 0 .or. bc_y%beg == -1) then
proc_coords(2) = proc_coords(2) - 1
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%beg, &
ierr)
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
bc_y%beg, ierr)
proc_coords(2) = proc_coords(2) + 1
end if
Expand All @@ -531,8 +533,8 @@ contains
! Boundary condition at the end
if (proc_coords(2) < num_procs_y - 1 .or. bc_y%end == -1) then
proc_coords(2) = proc_coords(2) + 1
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%end, &
ierr)
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
bc_y%end, ierr)
proc_coords(2) = proc_coords(2) - 1
end if
Expand Down Expand Up @@ -607,7 +609,7 @@ contains
else
offset_x%beg = 0
end if
! Boundary condition at the end
if (proc_coords(1) < num_procs_x - 1 .or. bc_x%end == -1) then
proc_coords(1) = proc_coords(1) + 1
Expand Down Expand Up @@ -928,7 +930,13 @@ contains
r = sys_size*(j + buff_size) &
+ sys_size*buff_size*k + (i - 1) &
+ sys_size*buff_size*(n + 1)*l
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down Expand Up @@ -1002,7 +1010,13 @@ contains
r = (i - 1) + sys_size*(j - m - 1) &
+ sys_size*buff_size*k &
+ sys_size*buff_size*(n + 1)*l
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down Expand Up @@ -1091,6 +1105,12 @@ contains
(k + buff_size) + sys_size* &
(m + 2*buff_size + 1)*buff_size*l
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down Expand Up @@ -1171,6 +1191,12 @@ contains
(k - n - 1) + sys_size* &
(m + 2*buff_size + 1)*buff_size*l
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down Expand Up @@ -1264,6 +1290,12 @@ contains
+ sys_size*(m + 2*buff_size + 1)* &
(n + 2*buff_size + 1)*(l + buff_size)
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down Expand Up @@ -1349,6 +1381,12 @@ contains
+ sys_size*(m + 2*buff_size + 1)* &
(n + 2*buff_size + 1)*(l - p - 1)
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
#if defined(__INTEL_COMPILER)
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
print *, "Error", j, k, l, i
error stop "NaN(s) in recv"
end if
#endif
end do
end do
end do
Expand Down
Loading

0 comments on commit 363d584

Please sign in to comment.