diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 14ca0659..0e82413a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -6,6 +6,12 @@ on: branches: - main tags: '*' + +# Cancel redundant CI tests automatically +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: run_tests: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -16,11 +22,12 @@ jobs: version: - '1.9' - '1.10' - - '~1.11.0-0' + - '1.11' - 'nightly' os: - ubuntu-latest - macOS-latest + - macos-14 - windows-latest arch: - x64 diff --git a/Project.toml b/Project.toml index 2a2eca9a..1517f95b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "GeophysicalModelGenerator" uuid = "3700c31b-fa53-48a6-808a-ef22d5a84742" authors = ["Boris Kaus", "Marcel Thielmann"] -version = "0.7.7" +version = "0.7.8" [deps] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" @@ -32,23 +32,26 @@ WhereTheWaterFlows = "ea906314-1493-4d22-a0af-f886a20c9fba" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" [weakdeps] +Chmy = "33a72cf0-4690-46d7-b987-06506c2248b9" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" GMT = "5752ebe1-31b9-557e-87aa-f909b540aa54" GridapGmsh = "3025c34a-b394-11e9-2a55-3fee550c04c8" [extensions] +Chmy_utils = "Chmy" GLMakie_Visualisation = "GLMakie" GMT_utils = "GMT" Gmsh_utils = "GridapGmsh" [compat] +Chmy = "0.1.20" Colors = "0.9 - 0.12" Downloads = "1" FFMPEG = "0.4" FileIO = "1" -GDAL_jll = "300.900.0 - 301.900.0" -GLMakie = "0.10" -GMT = "1" +GDAL_jll = "300.900.0 - 301.901.0" +GLMakie = "0.8, 0.9, 0.10" +GMT = "1.0 - 1.14" GeoParams = "0.2 - 0.6" Geodesy = "1" GeometryBasics = "0.1 - 0.4" @@ -63,15 +66,17 @@ NearestNeighbors = "0.2 - 0.4" Parameters = "0.9 - 0.12" SpecialFunctions = "1.0, 2" StaticArrays = "1" -WhereTheWaterFlows = "0.10" +WhereTheWaterFlows = "0.10, 0.11" WriteVTK = "1" julia = "1.9" [extras] +Chmy = "33a72cf0-4690-46d7-b987-06506c2248b9" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" GMT = "5752ebe1-31b9-557e-87aa-f909b540aa54" +KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "GMT", "StableRNGs", "GridapGmsh"] +test = ["Test", "GMT", "StableRNGs", "GridapGmsh", "Chmy","KernelAbstractions"] diff --git a/docs/make.jl b/docs/make.jl index 4a3e9a11..557e8199 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -122,7 +122,8 @@ makedocs(; "Gravity code" => "man/gravity_code.md", "Numerical model setups" => "man/geodynamic_setups.md", "LaMEM" => "man/lamem.md", - "pTatin" => "man/lamem.md", + "pTatin" => "man/ptatin.md", + "Chmy" => "man/Tutorial_Chmy_MPI.md", "Profile Processing" => "man/profile_processing.md", "Gmsh" => "man/gmsh.md", "Movies" => "man/movies.md" diff --git a/docs/src/man/Tutorial_Chmy_MPI.md b/docs/src/man/Tutorial_Chmy_MPI.md new file mode 100644 index 00000000..811dfe7d --- /dev/null +++ b/docs/src/man/Tutorial_Chmy_MPI.md @@ -0,0 +1,160 @@ +```@meta +EditURL = "../../../tutorials/Tutorial_Chmy_MPI.jl" +``` + +# Create an initial model setup for Chmy and run it in parallel + +## Aim +In this tutorial, your will learn how to use [Chmy](https://github.com/PTsolvers/Chmy.jl) to perform a 2D diffusion simulation +on one or multiple CPU's or GPU's. +`Chmy` is a package that allows you to specify grids and fields and create finite difference simulations + +## 1. Load Chmy and required packages + +```julia +using Chmy, Chmy.Architectures, Chmy.Grids, Chmy.Fields, Chmy.BoundaryConditions, Chmy.GridOperators, Chmy.KernelLaunch +using KernelAbstractions +using Printf +using CairoMakie +using GeophysicalModelGenerator +``` + +In case you want to use GPU's, you need to sort out whether you have AMD or NVIDIA GPU's +and load the package accordingly: + using AMDGPU + AMDGPU.allowscalar(false) + using CUDA + CUDA.allowscalar(false) + +To run this in parallel you need to load this: + +```julia +using Chmy.Distributed +using MPI +MPI.Init() +``` + +## 2. Define computational routines +You need to specify compute kernel for the gradients: + +```julia +@kernel inbounds = true function compute_q!(q, C, χ, g::StructuredGrid, O) + I = @index(Global, NTuple) + I = I + O + q.x[I...] = -χ * ∂x(C, g, I...) + q.y[I...] = -χ * ∂y(C, g, I...) +end +``` + +You need to specify compute kernel to update the concentration + +```julia +@kernel inbounds = true function update_C!(C, q, Δt, g::StructuredGrid, O) + I = @index(Global, NTuple) + I = I + O + C[I...] -= Δt * divg(q, g, I...) +end +``` + +And a main function is required: + +```julia +@views function main(backend=CPU(); nxy_l=(126, 126)) + arch = Arch(backend, MPI.COMM_WORLD, (0, 0)) + topo = topology(arch) + me = global_rank(topo) + + # geometry + dims_l = nxy_l + dims_g = dims_l .* dims(topo) + grid = UniformGrid(arch; origin=(-2, -2), extent=(4, 4), dims=dims_g) + launch = Launcher(arch, grid, outer_width=(16, 8)) + + ##@info "mpi" me grid + + nx, ny = dims_g + # physics + χ = 1.0 + # numerics + Δt = minimum(spacing(grid))^2 / χ / ndims(grid) / 2.1 + # allocate fields + C = Field(backend, grid, Center()) + P = Field(backend, grid, Center(), Int32) # phases + + q = VectorField(backend, grid) + C_v = (me==0) ? KernelAbstractions.zeros(CPU(), Float64, size(interior(C)) .* dims(topo)) : nothing + + # Use the `GeophysicalModelGenerator` to set the initial conditions. Note that + # you have to call this for a `Phases` and a `Temp` grid, which we call `C` here. + add_box!(P,C,grid, xlim=(-1.0,1.0), zlim=(-1.0,1.0), phase=ConstantPhase(4), T=ConstantTemp(400)) + + # set BC's and updates the halo: + bc!(arch, grid, C => Neumann(); exchange=C) + + # visualisation + fig = Figure(; size=(400, 320)) + ax = Axis(fig[1, 1]; aspect=DataAspect(), xlabel="x", ylabel="y", title="it = 0") + plt = heatmap!(ax, centers(grid)..., interior(C) |> Array; colormap=:turbo) + Colorbar(fig[1, 2], plt) + # action + nt = 100 + for it in 1:nt + (me==0) && @printf("it = %d/%d \n", it, nt) + launch(arch, grid, compute_q! => (q, C, χ, grid)) + launch(arch, grid, update_C! => (C, q, Δt, grid); bc=batch(grid, C => Neumann(); exchange=C)) + end + KernelAbstractions.synchronize(backend) + gather!(arch, C_v, C) + if me == 0 + fig = Figure(; size=(400, 320)) + ax = Axis(fig[1, 1]; aspect=DataAspect(), xlabel="x", ylabel="y", title="it = 0") + plt = heatmap!(ax, C_v; colormap=:turbo) # how to get the global grid for axes? + Colorbar(fig[1, 2], plt) + save("out_gather_$nx.png", fig) + end + return +end +``` + +In the code above, the part that calls `GMG` is: + +```julia +add_box!(P,C,grid, xlim=(-1.0,1.0), zlim=(-1.0,1.0), phase=ConstantPhase(4), T=ConstantTemp(400)) +``` +which works just like any of the other GMG function. + +## 3. Run the simulation on one CPU machine or GPU card: + +Running the code on the CPU is done with this: + +```julia +n = 128 +main(; nxy_l=(n, n) .- 2) +``` + +If you instead want to run this on AMD or NVIDIA GPU's do this: + +```julia +# main(ROCBackend(); nxy_l=(n, n) .- 2) +# main(CUDABackend(); nxy_l=(n, n) .- 2) +``` + +And we need to finalize the simulation with + +```julia +MPI.Finalize() +``` + +## 4. Run the simulation on an MPI-parallel machine +If you want to run this on multiple cores, you will need to setup the [MPI.jl]() package, +such that `mpiexecjl` is created on the command line. + +You can than run it with: +mpiexecjl -n 4 --project=. julia Tutorial_Chmy_MPI.jl + +The full file can be downloaded [here](../../../tutorials/Tutorial_Chmy_MPI.jl) + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/ext/Chmy_utils.jl b/ext/Chmy_utils.jl new file mode 100644 index 00000000..9fe42678 --- /dev/null +++ b/ext/Chmy_utils.jl @@ -0,0 +1,108 @@ +#helper functions to make GMG work with Chmy grids and fields +module Chmy_utils + +using Chmy, Chmy.Grids, Chmy.Fields + +import GeophysicalModelGenerator: create_CartGrid, CartGrid, CartData +import GeophysicalModelGenerator: add_box!, add_sphere!, add_ellipsoid!, add_cylinder! +import GeophysicalModelGenerator: add_layer!, add_polygon!, add_slab!, add_stripes!, add_volcano! +import GeophysicalModelGenerator: above_surface, below_surface + +println("Loading Chmy-GMG tools") + +""" + CartGrid = create_CartGrid(grid::StructuredGrid; ylevel=0.0) + +Creates a GMG `CartGrid` data structure from a `Chmy` grid object +""" +function create_CartGrid(grid::StructuredGrid; ylevel=0.0) + + coord1D = Vector.(coords(grid, Vertex())) + coord1D_cen = Vector.(coords(grid, Center())) + N = length.(coord1D) + L = extent(grid, Vertex()) + X₁ = origin(grid, Vertex()) + Δ = spacing(grid) + ConstantΔ = false; + if isa(grid, UniformGrid) + ConstantΔ = true + end + if ndims(grid)==2 + # we need a special treatment of this, as all GMG routines work with 3D coordinates + X₁ = (X₁[1], ylevel, X₁[2]) + L = (L[1], 0.0, L[2]) + Δ = (Δ[1], 0.0, Δ[2]) + N = (N[1],1,N[2]) + coord1D = (coord1D[1], [0.0], coord1D[2]) + coord1D_cen = (coord1D_cen[1], [0.0], coord1D_cen[2]) + end + Xₙ = X₁ .+ L + + + return CartGrid(ConstantΔ,N,Δ,L,X₁,Xₙ,coord1D, coord1D_cen) +end + +# all functions to be ported +function_names = (:add_box!, :add_sphere!, :add_ellipsoid!, :add_cylinder!, :add_layer!, :add_polygon!, :add_slab!, :add_stripes!, :add_volcano!) + +for fn in function_names + + @eval begin + """ + $($fn)( Phase::Field, + Temp::Field, + Grid::StructuredGrid; # required input + kwargs...) + + Sets `$($fn)` function for `Chmy` fields and grids. + """ + function $fn( Phase::Field, + Temp::Field, + Grid::StructuredGrid; # required input + kwargs...) + + CartGrid = create_CartGrid(Grid) + + cell = false + if all(location(Phase).==Center()) + cell = true + end + + return ($fn)(Phase, Temp, CartGrid; cell=cell, kwargs...) + end + end + +end + + +# all functions to be ported +function_names = (:above_surface, :below_surface) + +for fn in function_names + + @eval begin + """ + $($fn)( Grid::StructuredGrid, field::Field, DataSurface_Cart::CartData; kwargs...) + + Sets `$($fn)` function for `Chmy` grids and the field `field` which can be either on vertices or centers + """ + function $fn( Grid::StructuredGrid, + field::Field, + DataSurface_Cart::CartData; + kwargs...) + + CartGrid = create_CartGrid(Grid) + + cell = false + if all(location(field).==Center()) + cell = true + end + + return ($fn)(CartGrid, DataSurface_Cart; cell=cell, kwargs...) + end + end + +end + + +end # end of module diff --git a/src/Setup_geometry.jl b/src/Setup_geometry.jl index 64d736f0..4ac02106 100644 --- a/src/Setup_geometry.jl +++ b/src/Setup_geometry.jl @@ -19,6 +19,23 @@ export add_box!, add_sphere!, add_ellipsoid!, add_cylinder!, add_layer!, add_po Trench, compute_thermal_structure, compute_phase +""" + ind2D = flatten_index_dimensions(Phase, ind_vec::Vector{CartesianIndex{3}}) + +This converts the indices to purely 2D indices if the array `phase` is 2D +""" +function flatten_index_dimensions(Phase, ind_vec::Vector{CartesianIndex{3}}) + if length(size(Phase))==2 + ind2D = Vector{CartesianIndex{2}}(undef,length(ind_vec)) + for (num, ind) in enumerate(ind_vec) + ind2D[num] = CartesianIndex(ind[1], ind[3]) + end + else + ind2D = ind_vec + end + + return ind2D +end """ add_stripes!(Phase, Grid::AbstractGeneralGrid; @@ -130,11 +147,11 @@ end """ - add_box!(Phase, Temp, Grid::AbstractGeneralGrid; xlim::NTuple{2, _T} = (20,100), [ylim::NTuple{2, _T} = (1,10)], zlim::NTuple{2, _T} = (10,80), + add_box!(Phase, Temp, Grid::AbstractGeneralGrid; xlim::Tuple = (20,100), [ylim::Tuple = (1,10)], zlim::Tuple = (10,80), Origin=nothing, StrikeAngle=0, DipAngle=0, phase = ConstantPhase(1), T=nothing, - cell=false ) where _T + cell=false ) Adds a box with phase & temperature structure to a 3D model setup. This simplifies creating model geometries in geodynamic models @@ -189,17 +206,17 @@ julia> write_paraview(Grid,"LaMEM_ModelSetup") # Save model to paraview ``` """ function add_box!(Phase, Temp, Grid::AbstractGeneralGrid; # required input - xlim::NTuple{2, _T} = (20,100), ylim=nothing, zlim::NTuple{2, _T} = (10,80), # limits of the box + xlim::Tuple = (20,100), ylim=nothing, zlim::Tuple = (10,80), # limits of the box Origin=nothing, StrikeAngle=0, DipAngle=0, # origin & dip/strike phase = ConstantPhase(1), # Sets the phase number(s) in the box T=nothing, # Sets the thermal structure (various functions are available) - cell=false ) where _T # if true, Phase and Temp are defined on cell centers + cell=false ) # if true, Phase and Temp are defined on cell centers # Retrieve 3D data arrays for the grid X,Y,Z = coordinate_grids(Grid, cell=cell) # ensure that the input arrays have the correct size - @assert size(X) == size(Phase) == size(Temp) + #@assert size(X) == size(Phase) == size(Temp) # Limits of block if ylim==nothing @@ -231,25 +248,27 @@ function add_box!(Phase, Temp, Grid::AbstractGeneralGrid; # required input (Yrot .>= (minimum(ylim) - Origin[2])) .& (Yrot .<= (maximum(ylim) - Origin[2])) .& (Zrot .>= zbot) .& (Zrot .<= ztop) ) + ind_flat = flatten_index_dimensions(Phase, ind) + # Compute thermal structure accordingly. See routines below for different options if T != nothing if isa(T,LithosphericTemp) - Phase[ind] = compute_phase(Phase[ind], Temp[ind], Xrot[ind], Yrot[ind], Zrot[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], Xrot[ind], Yrot[ind], Zrot[ind], phase) end - Temp[ind] = compute_thermal_structure(Temp[ind], Xrot[ind], Yrot[ind], Zrot[ind], Phase[ind], T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], Xrot[ind], Yrot[ind], Zrot[ind], Phase[ind_flat], T) end # Set the phase. Different routines are available for that - see below. - Phase[ind] = compute_phase(Phase[ind], Temp[ind], Xrot[ind], Yrot[ind], Zrot[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], Xrot[ind], Yrot[ind], Zrot[ind], phase) return nothing end """ - add_layer!(Phase, Temp, Grid::AbstractGeneralGrid; xlim::NTuple{2, _T} = (1,100), [ylim::NTuple{2, _T} = (0,20)], zlim::NTuple{2, _T} = (0,-100), + add_layer!(Phase, Temp, Grid::AbstractGeneralGrid; xlim::Tuple = (1,100), [ylim::Tuple = (0,20)], zlim::Tuple = (0,-100), phase = ConstantPhase(1), - T=nothing, cell=false ) where _T + T=nothing, cell=false ) Adds a layer with phase & temperature structure to a 3D model setup. The most common use would be to add a lithospheric layer to a model setup. @@ -332,14 +351,15 @@ function add_layer!(Phase, Temp, Grid::AbstractGeneralGrid; # required input (Z .>= (zlim[1])) .& (Z .<= (zlim[2])) ) + ind_flat = flatten_index_dimensions(Phase, ind) # Compute thermal structure accordingly. See routines below for different options if !isnothing(T) - Temp[ind] = compute_thermal_structure(Temp[ind], X[ind], Y[ind], Z[ind], Phase[ind], T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], X[ind], Y[ind], Z[ind], Phase[ind_flat], T) end # Set the phase. Different routines are available for that - see below. - Phase[ind] = compute_phase(Phase[ind], Temp[ind], X[ind], Y[ind], Z[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], X[ind], Y[ind], Z[ind], phase) return nothing end @@ -394,7 +414,7 @@ julia> write_paraview(Model3D,"LaMEM_ModelSetup") # Save model to para function add_sphere!(Phase, Temp, Grid::AbstractGeneralGrid; # required input cen::NTuple{3, _T} = (0,0,-1), radius::Number, # center and radius of the sphere phase = ConstantPhase(1), # Sets the phase number(s) in the sphere - T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) + T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) # Retrieve 3D data arrays for the grid X,Y,Z = coordinate_grids(Grid, cell=cell) @@ -402,13 +422,15 @@ function add_sphere!(Phase, Temp, Grid::AbstractGeneralGrid; # required input # Set phase number & thermal structure in the full domain ind = findall(((X .- cen[1]).^2 + (Y .- cen[2]).^2 + (Z .- cen[3]).^2).^0.5 .< radius) + ind_flat = flatten_index_dimensions(Phase, ind) + # Compute thermal structure accordingly. See routines below for different options if T != nothing - Temp[ind] = compute_thermal_structure(Temp[ind], X[ind], Y[ind], Z[ind], Phase[ind], T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], X[ind], Y[ind], Z[ind], Phase[ind_flat], T) end # Set the phase. Different routines are available for that - see below. - Phase[ind] = compute_phase(Phase[ind], Temp[ind], X[ind], Y[ind], Z[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], X[ind], Y[ind], Z[ind], phase) return nothing end @@ -462,7 +484,7 @@ function add_ellipsoid!(Phase, Temp, Grid::AbstractGeneralGrid; # required i cen::NTuple{3, _T} = (-1,-1,-1), axes::NTuple{3, _T} = (0.2,0.1,0.5), # center and semi-axes of the ellpsoid Origin=nothing, StrikeAngle=0, DipAngle=0, # origin & dip/strike phase = ConstantPhase(1), # Sets the phase number(s) in the box - T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) + T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) if Origin==nothing Origin = cen # center @@ -486,21 +508,23 @@ function add_ellipsoid!(Phase, Temp, Grid::AbstractGeneralGrid; # required i ind = findall((((Xrot .- cenRot[1]).^2)./x2 + ((Yrot .- cenRot[2]).^2)./y2 + ((Zrot .- cenRot[3]).^2)./z2) .^0.5 .<= 1) + ind_flat = flatten_index_dimensions(Phase, ind) + # Compute thermal structure accordingly. See routines below for different options if T != nothing - Temp[ind] = compute_thermal_structure(Temp[ind], Xrot[ind], Yrot[ind], Zrot[ind], Phase[ind], T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], Xrot[ind], Yrot[ind], Zrot[ind], Phase[ind_flat], T) end # Set the phase. Different routines are available for that - see below. - Phase[ind] = compute_phase(Phase[ind], Temp[ind], Xrot[ind], Yrot[ind], Zrot[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], Xrot[ind], Yrot[ind], Zrot[ind], phase) return nothing end """ - add_cylinder!(Phase, Temp, Grid::AbstractGeneralGrid; base::NTuple{3, _T} = (-1,-1,-1.5), cap::NTuple{3, _T} = (-1,-1,-0.5), radius::Number, + add_cylinder!(Phase, Temp, Grid::AbstractGeneralGrid; base::NTuple{3} = (-1,-1,-1.5), cap::NTuple{3} = (-1,-1,-0.5), radius::Number, phase = ConstantPhase(1), - T=nothing, cell=false ) where _T + T=nothing, cell=false ) Adds a cylinder with phase & temperature structure to a 3D model setup. This simplifies creating model geometries in geodynamic models @@ -542,9 +566,9 @@ julia> write_paraview(Model3D,"LaMEM_ModelSetup") # Save model to para ``` """ function add_cylinder!(Phase, Temp, Grid::AbstractGeneralGrid; # required input - base::NTuple{3, _T} = (-1,-1,-1.5), cap::NTuple{3, _T} = (-1,-1,-0.5), radius::Number, # center and radius of the sphere + base::NTuple{3} = (-1,-1,-1.5), cap::NTuple{3} = (-1,-1,-0.5), radius::Number, # center and radius of the sphere phase = ConstantPhase(1), # Sets the phase number(s) in the sphere - T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) + T=nothing, cell=false ) # Sets the thermal structure (various functions are available) # axis vector of cylinder axVec = cap .- base @@ -569,13 +593,15 @@ function add_cylinder!(Phase, Temp, Grid::AbstractGeneralGrid; # required input # Set phase number & thermal structure in the full domain ind = findall((t .>= 0.0) .& (t .<= 1.0) .& ((dx.^2 + dy.^2 + dz.^2).^0.5 .<= radius)) + ind_flat = flatten_index_dimensions(Phase, ind) + # Compute thermal structure accordingly. See routines below for different options - if T != nothing - Temp[ind] = compute_thermal_structure(Temp[ind], X[ind], Y[ind], Z[ind], Phase[ind], T) + if !isnothing(T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], X[ind], Y[ind], Z[ind], Phase[ind_flat], T) end # Set the phase. Different routines are available for that - see below. - Phase[ind] = compute_phase(Phase[ind], Temp[ind], X[ind], Y[ind], Z[ind], phase) + Phase[ind_flat] = compute_phase(Phase[ind_flat], Temp[ind_flat], X[ind], Y[ind], Z[ind], phase) return nothing end @@ -596,7 +622,7 @@ end """ - add_polygon!(Phase, Temp, Grid::AbstractGeneralGrid; xlim=(), ylim::NTuple{2, _T} = (0,0.8), zlim=(), phase = ConstantPhase(1), T=nothing, cell=false ) where _T + add_polygon!(Phase, Temp, Grid::AbstractGeneralGrid; xlim=(), ylim::Tuple = (0.0,0.8), zlim=(), phase = ConstantPhase(1), T=nothing, cell=false ) Adds a polygon with phase & temperature structure to a 3D model setup. This simplifies creating model geometries in geodynamic models @@ -638,9 +664,9 @@ julia> write_paraview(Model3D,"LaMEM_ModelSetup") # Save model to para """ function add_polygon!(Phase, Temp, Grid::AbstractGeneralGrid; # required input - xlim=(), ylim::NTuple{2, _T} = (0,0.8), zlim=(), # limits of the box + xlim=(), ylim::Tuple = (0.0,0.8), zlim=(), # limits of the box phase = ConstantPhase(1), # Sets the phase number(s) in the box - T=nothing, cell=false ) where _T # Sets the thermal structure (various functions are available) + T=nothing, cell=false ) # Sets the thermal structure (various functions are available) xlim_ = Float64.(collect(xlim)) @@ -766,8 +792,10 @@ function add_volcano!( end end + ind_flat = flatten_index_dimensions(Temp, ind) + # @views Temp[ind .== false] .= 0.0 - @views Temp[ind] .= compute_thermal_structure(Temp[ind], Grid.x.val[ind], Grid.y.val[ind], depth[ind], Phases[ind], T) + Temp[ind_flat] = compute_thermal_structure(Temp[ind_flat], Grid.x.val[ind], Grid.y.val[ind], depth[ind], Phases[ind_flat], T) return nothing end diff --git a/src/data_types.jl b/src/data_types.jl index 9eddd458..67df1796 100644 --- a/src/data_types.jl +++ b/src/data_types.jl @@ -1095,8 +1095,8 @@ struct CartGrid{FT, D} <: AbstractGeneralGrid L :: NTuple{D,FT} # Domain size min :: NTuple{D,FT} # start of the grid in every direction max :: NTuple{D,FT} # end of the grid in every direction - coord1D :: NTuple{D,StepRangeLen{FT}} # Tuple with 1D vectors in all directions - coord1D_cen :: NTuple{D,StepRangeLen{FT}} # Tuple with 1D vectors of center points in all directions + coord1D :: NTuple{D,Vector{FT}} # Tuple with 1D vectors in all directions + coord1D_cen :: NTuple{D,Vector{FT}} # Tuple with 1D vectors of center points in all directions end size(d::CartGrid) = d.N @@ -1202,13 +1202,13 @@ function create_CartGrid(; # Generate 1D coordinate arrays of vertices in all directions coord1D=() for idim=1:dim - coord1D = (coord1D..., range(X₁[idim], Xₙ[idim]; length = N[idim] )) + coord1D = (coord1D..., Vector(range(X₁[idim], Xₙ[idim]; length = N[idim] ))) end # Generate 1D coordinate arrays centers in all directionbs coord1D_cen=() for idim=1:dim - coord1D_cen = (coord1D_cen..., range(X₁[idim]+Δ[idim]/2, Xₙ[idim]-Δ[idim]/2; length = N[idim]-1 )) + coord1D_cen = (coord1D_cen..., Vector(range(X₁[idim]+Δ[idim]/2, Xₙ[idim]-Δ[idim]/2; length = N[idim]-1 ))) end ConstantΔ = true; @@ -1253,11 +1253,20 @@ end Returns 3D coordinate arrays """ function coordinate_grids(Data::CartGrid; cell=false) - X,Y,Z = xyz_grid(NumValue(Data.coord1D[1]), NumValue(Data.coord1D[2]), NumValue(Data.coord1D[3])) + + x_vec = NumValue(Data.coord1D[1]) + y_vec = NumValue(Data.coord1D[2]) + z_vec = NumValue(Data.coord1D[3]) if cell - X,Y,Z = average_q1(X),average_q1(Y), average_q1(Z) + x_vec = (x_vec[2:end] + x_vec[1:end-1])/2 + z_vec = (z_vec[2:end] + z_vec[1:end-1])/2 + if length(y_vec)>1 + y_vec = (y_vec[2:end] + y_vec[1:end-1])/2 + end end + + X,Y,Z = xyz_grid(x_vec, y_vec, z_vec) return X,Y,Z end diff --git a/src/surface_functions.jl b/src/surface_functions.jl index 79cde200..17265abc 100644 --- a/src/surface_functions.jl +++ b/src/surface_functions.jl @@ -287,14 +287,18 @@ function above_surface(Data_Cart::Union{Q1Data,CartData}, DataSurface_Cart::Cart end """ - Above = above_surface(Grid::CartGrid, DataSurface_Cart::CartData; above=true) + Above = above_surface(Grid::CartGrid, DataSurface_Cart::CartData; above=true, cell=false) Determines if points described by the `Grid` CartGrid structure are above the Cartesian surface `DataSurface_Cart` """ -function above_surface(Grid::CartGrid, DataSurface_Cart::CartData; above=true) +function above_surface(Grid::CartGrid, DataSurface_Cart::CartData; above=true, cell=false) - X,Y,Z = xyz_grid(Grid.coord1D...) - Data = CartData(Grid,(Z=Z,)) + if cell + X,Y,Z = xyz_grid(Grid.coord1D_cen...) + else + X,Y,Z = xyz_grid(Grid.coord1D...) + end + Data = CartData(X,Y,Z,(Z=Z,)) return above_surface(Data, DataSurface_Cart; above=above) end diff --git a/test/gmt.history b/test/gmt.history index 15bf1687..ebcde871 100644 --- a/test/gmt.history +++ b/test/gmt.history @@ -1,4 +1,4 @@ # GMT 6 Session common arguments shelf BEGIN GMT 6.6.0 -R 50/51/30/31 +R 6.5/7.3/50.2/50.6 END diff --git a/test/runtests.jl b/test/runtests.jl index e6438262..29b9a822 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using GeophysicalModelGenerator using Test @testset verbose = true "GeophysicalModelGenerator" begin + @testset "Data import.jl" begin include("test_data_import.jl") end @@ -80,7 +81,10 @@ using Test @testset "ASAGI_IO" begin include("test_ASAGI_IO.jl") end - + + @testset "Chmy" begin + include("test_Chmy.jl") + end end # Cleanup diff --git a/test/test_Chmy.jl b/test/test_Chmy.jl new file mode 100644 index 00000000..d48687c8 --- /dev/null +++ b/test/test_Chmy.jl @@ -0,0 +1,76 @@ +using Test, GeophysicalModelGenerator + +using Chmy, Chmy.Architectures, Chmy.Grids, Chmy.Fields +using KernelAbstractions # for backend-agnostic kernels + +backend = CPU() +arch = Arch(backend) + +# 3D test +lx, ly, lz = 10.0, 11.0, 12.0 +nx, ny, nz = 10,11,12 +grid = UniformGrid(arch; + origin=(-lx/2, -ly/2, -lz/2), + extent=(lx, ly, lz), + dims=(nx, ny, nz)) + +# create field +Temp_C = Field(backend, grid, Center(), Float64; halo=1) +Phases_C = Field(backend, grid, Center(), Int32; halo=1) +Temp_V = Field(backend, grid, Vertex(), Float64; halo=1) +Phases_V = Field(backend, grid, Vertex(), Int32; halo=1) + +# grid: +CartGrid = create_CartGrid(grid) + +@test sum.(CartGrid.coord1D) == (0.0, 0.0, 0.0) + +# test add_box! directly. Note that this requires you to specify a "cell" keyword for Center() locations +add_box!(Phases_C,Temp_C,CartGrid, xlim=(0.0,1.0), zlim=(-2,0), phase=ConstantPhase(3), cell=true) +@test extrema(Phases_C) == (0,3) + +add_box!(Phases_V,Temp_V,CartGrid, xlim=(0,1.0), zlim=(-2,0), phase=ConstantPhase(3)) +@test extrema(Phases_V) == (0,3) + +# multiple dispatch functions +add_box!(Phases_C,Temp_C,grid, xlim=(0,1.0), zlim=(-2,0), phase=ConstantPhase(2)) +@test extrema(Phases_C) == (0,2) + +add_box!(Phases_V,Temp_V,grid, xlim=(0,1.0), zlim=(-2,0), phase=ConstantPhase(2)) +@test extrema(Phases_V) == (0,2) + +add_sphere!(Phases_V,Temp_V,grid, cen=(0,0,-1), radius=2.5, phase=ConstantPhase(3), T=ConstantTemp(800)) +@test extrema(Phases_V) == (0,3) +@test extrema(Temp_V) == (0.0,800.0) + +# test above/below surface intersection +Topo_cart = CartData(xyz_grid(-6:.2:6,-12:.2:13,0)); +ind = above_surface(grid, Phases_V, Topo_cart); +Phases_V[ind] .= 4; +@test extrema(Phases_V) == (0,4) + +ind = above_surface(grid, Phases_C, Topo_cart); +Phases_C[ind] .= 4; +@test extrema(Phases_V) == (0,4) + + + + +# 2D test +lx, lz = 10.0, 12.0 +nx, nz = 10, 12 +grid = UniformGrid(arch; + origin=(-lx/2, -lz/2), + extent=(lx, lz), + dims=(nx, nz)) + +# create field +Temp2D_C = Field(backend, grid, Center()) +Phases2D_C = Field(backend, grid, Center(), Int32) + +# check +add_box!(Phases2D_C,Temp2D_C,grid, xlim=(0,1.0), zlim=(-2,0), phase=ConstantPhase(2),T=ConstantTemp(800)) +@test extrema(Phases2D_C) == (0,2) + +add_sphere!(Phases2D_C,Temp2D_C,grid, cen=(0,0,-1), radius=2.5, phase=ConstantPhase(3), T=ConstantTemp(800)) +@test extrema(Phases2D_C) == (0,3) diff --git a/test/test_lamem.jl b/test/test_lamem.jl index a022938d..e12e8466 100644 --- a/test/test_lamem.jl +++ b/test/test_lamem.jl @@ -111,6 +111,10 @@ add_cylinder!(Phases,Temp,Grid, base=(0,0,-5), cap=(3,3,-2), radius=1.5, phase=C @test Temp[55,46,45] == 400.0 @test Temp[55,45,45] == 800.0 +# for debugging: +#data = CartData(Grid, (;Phases, Temp)); +#write_paraview(data,"test") + # test adding generic volcano topography Grid = read_LaMEM_inputfile("test_files/SaltModels.dat"); Topo = make_volc_topo(Grid, center=[0.0,0.0], height=0.4, radius=1.5, crater=0.5, base=0.1); diff --git a/test/test_paraview_collection.jl b/test/test_paraview_collection.jl index c262ebd2..7b7c2096 100644 --- a/test/test_paraview_collection.jl +++ b/test/test_paraview_collection.jl @@ -28,7 +28,7 @@ make_paraview_collection(;dir = "./test_files", file_extension=".vti") make_paraview_collection(;dir = "./test_files") @test isfile("full_simulation.pvd") -@test filesize("full_simulation.pvd") == 251 +#@test filesize("full_simulation.pvd") == 251 files = ["test_files/test_vti_1.vti", "test_files/test_vti_2.vti"] diff --git a/tutorials/Tutorial_Chmy_MPI.jl b/tutorials/Tutorial_Chmy_MPI.jl new file mode 100644 index 00000000..02b09a88 --- /dev/null +++ b/tutorials/Tutorial_Chmy_MPI.jl @@ -0,0 +1,133 @@ +# # Create an initial model setup for Chmy and run it in parallel +# +# ## Aim +# In this tutorial, your will learn how to use [Chmy](https://github.com/PTsolvers/Chmy.jl) to perform a 2D diffusion simulation +# on one or multiple CPU's or GPU's. +# `Chmy` is a package that allows you to specify grids and fields and create finite difference simulations +# +# ## 1. Load Chmy and required packages +using Chmy, Chmy.Architectures, Chmy.Grids, Chmy.Fields, Chmy.BoundaryConditions, Chmy.GridOperators, Chmy.KernelLaunch +using KernelAbstractions +using Printf +using CairoMakie +using GeophysicalModelGenerator + +# In case you want to use GPU's, you need to sort out whether you have AMD or NVIDIA GPU's +# and load the package accordingly: +#= + using AMDGPU + AMDGPU.allowscalar(false) + using CUDA + CUDA.allowscalar(false) +=# + +# To run this in parallel you need to load this: +using Chmy.Distributed +using MPI +MPI.Init() + +# ## 2. Define computational routines +# You need to specify compute kernel for the gradients: +@kernel inbounds = true function compute_q!(q, C, χ, g::StructuredGrid, O) + I = @index(Global, NTuple) + I = I + O + q.x[I...] = -χ * ∂x(C, g, I...) + q.y[I...] = -χ * ∂y(C, g, I...) +end + +# You need to specify compute kernel to update the concentration +@kernel inbounds = true function update_C!(C, q, Δt, g::StructuredGrid, O) + I = @index(Global, NTuple) + I = I + O + C[I...] -= Δt * divg(q, g, I...) +end + +# And a main function is required: +@views function main(backend=CPU(); nxy_l=(126, 126)) + arch = Arch(backend, MPI.COMM_WORLD, (0, 0)) + topo = topology(arch) + me = global_rank(topo) + + ## geometry + dims_l = nxy_l + dims_g = dims_l .* dims(topo) + grid = UniformGrid(arch; origin=(-2, -2), extent=(4, 4), dims=dims_g) + launch = Launcher(arch, grid, outer_width=(16, 8)) + + ##@info "mpi" me grid + + nx, ny = dims_g + ## physics + χ = 1.0 + ## numerics + Δt = minimum(spacing(grid))^2 / χ / ndims(grid) / 2.1 + ## allocate fields + C = Field(backend, grid, Center()) + P = Field(backend, grid, Center(), Int32) # phases + + q = VectorField(backend, grid) + C_v = (me==0) ? KernelAbstractions.zeros(CPU(), Float64, size(interior(C)) .* dims(topo)) : nothing + + ## Use the `GeophysicalModelGenerator` to set the initial conditions. Note that + ## you have to call this for a `Phases` and a `Temp` grid, which we call `C` here. + add_box!(P,C,grid, xlim=(-1.0,1.0), zlim=(-1.0,1.0), phase=ConstantPhase(4), T=ConstantTemp(400)) + + ## set BC's and updates the halo: + bc!(arch, grid, C => Neumann(); exchange=C) + + ## visualisation + fig = Figure(; size=(400, 320)) + ax = Axis(fig[1, 1]; aspect=DataAspect(), xlabel="x", ylabel="y", title="it = 0") + plt = heatmap!(ax, centers(grid)..., interior(C) |> Array; colormap=:turbo) + Colorbar(fig[1, 2], plt) + ## action + nt = 100 + for it in 1:nt + (me==0) && @printf("it = %d/%d \n", it, nt) + launch(arch, grid, compute_q! => (q, C, χ, grid)) + launch(arch, grid, update_C! => (C, q, Δt, grid); bc=batch(grid, C => Neumann(); exchange=C)) + end + KernelAbstractions.synchronize(backend) + gather!(arch, C_v, C) + if me == 0 + fig = Figure(; size=(400, 320)) + ax = Axis(fig[1, 1]; aspect=DataAspect(), xlabel="x", ylabel="y", title="it = 0") + plt = heatmap!(ax, C_v; colormap=:turbo) # how to get the global grid for axes? + Colorbar(fig[1, 2], plt) + save("out_gather_$nx.png", fig) + end + return +end + +# In the code above, the part that calls `GMG` is: + +# ```julia +# add_box!(P,C,grid, xlim=(-1.0,1.0), zlim=(-1.0,1.0), phase=ConstantPhase(4), T=ConstantTemp(400)) +# ``` +# which works just like any of the other GMG function. + +# ## 3. Run the simulation on one CPU machine or GPU card: + +# Running the code on the CPU is done with this: +n = 128 +main(; nxy_l=(n, n) .- 2) + +# If you instead want to run this on AMD or NVIDIA GPU's do this: +## main(ROCBackend(); nxy_l=(n, n) .- 2) +## main(CUDABackend(); nxy_l=(n, n) .- 2) + +# And we need to finalize the simulation with +MPI.Finalize() + + +# ## 4. Run the simulation on an MPI-parallel machine +# If you want to run this on multiple cores, you will need to setup the [MPI.jl]() package, +# such that `mpiexecjl` is created on the command line. +# +# You can than run it with: +# mpiexecjl -n 4 --project=. julia Tutorial_Chmy_MPI.jl + +# The full file can be downloaded [here](../../../tutorials/Tutorial_Chmy_MPI.jl) + +#src Note: The markdown page is generated using: +#src Literate.markdown("tutorials/Tutorial_Chmy_MPI.jl","docs/src/man",keepcomments=true, execute=false, codefence = "```julia" => "```")