diff --git a/.build_adios2_for_ci.sh b/.build_adios2_for_ci.sh new file mode 100755 index 0000000000..c6d4178884 --- /dev/null +++ b/.build_adios2_for_ci.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +if test $BUILD_ADIOS2 ; then + if [[ ! -d $HOME/local/adios/include/adios2.h ]] || test $1 ; then + echo "****************************************" + echo "Building ADIOS2" + echo "****************************************" + + branch=${1:-release_29} + if [ ! -d adios2 ]; then + git clone -b $branch https://github.com/ornladios/ADIOS2.git adios2 --depth=1 + fi + + pushd adios2 + rm -rf build + mkdir -p build + pushd build + + cmake .. \ + -DCMAKE_INSTALL_PREFIX=$HOME/local \ + -DADIOS2_USE_MPI=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_TESTING=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_MGARD=OFF \ + -DADIOS2_USE_HDF5=OFF \ + -DADIOS2_USE_BZip2=OFF \ + -DADIOS2_USE_Blosc2=OFF \ + -DADIOS2_USE_SZ=OFF \ + -DADIOS2_USE_ZFP=OFF \ + -DADIOS2_USE_DAOS=OFF \ + -DADIOS2_USE_UCX=OFF \ + -DADIOS2_USE_LIBPRESSIO=OFF \ + -DADIOS2_USE_Sodium=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DADIOS2_USE_MHS=OFF \ + -DADIOS2_USE_DataMan=OFF + + make -j 4 && make install + popd + + echo "****************************************" + echo " Finished building ADIOS2" + echo "****************************************" + + else + + echo "****************************************" + echo " ADIOS2 already installed" + echo "****************************************" + fi +else + echo "****************************************" + echo " ADIOS2 not requested" + echo "****************************************" +fi diff --git a/.ci_fedora.sh b/.ci_fedora.sh index 0774000b9c..452afb4b7e 100755 --- a/.ci_fedora.sh +++ b/.ci_fedora.sh @@ -56,6 +56,7 @@ else . /etc/profile.d/modules.sh module load mpi/${1}-x86_64 export OMPI_MCA_rmaps_base_oversubscribe=yes + export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe export TRAVIS=true export FLEXIBLAS=NETLIB cd diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6aaedb5804..c449beb4ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,7 @@ jobs: OMP_NUM_THREADS: ${{ matrix.config.omp_num_threads }} PYTHONPATH: ${{ github.workspace }}/tools/pylib OMPI_MCA_rmaps_base_oversubscribe: yes + PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe" MPIRUN: mpiexec -np strategy: fail-fast: true @@ -38,7 +39,7 @@ jobs: is_cron: - ${{ github.event_name == 'cron' }} config: - - name: "CMake, PETSc unreleased" + - name: "CMake, PETSc unreleased, ADIOS" os: ubuntu-20.04 cmake_options: "-DBUILD_SHARED_LIBS=ON -DBOUT_ENABLE_METRIC_3D=ON @@ -46,12 +47,15 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_ADIOS2=ON -DBOUT_ENABLE_PYTHON=ON + -DADIOS2_ROOT=/home/runner/local/adios2 -DSUNDIALS_ROOT=/home/runner/local -DPETSC_DIR=/home/runner/local/petsc -DSLEPC_DIR=/home/runner/local/slepc" build_petsc: -petsc-main build_petsc_branch: main + build_adios2: true on_cron: true - name: "Default options, Ubuntu 20.04" @@ -185,9 +189,8 @@ jobs: - name: Install pip packages run: | - ./.pip_install_for_ci.sh 'cython~=0.29' 'netcdf4~=1.5' 'sympy~=1.5' 'gcovr' 'cmake' zoidberg fastcov - # Add the pip install location to the runner's PATH - echo ~/.local/bin >> $GITHUB_PATH + python -m pip install --upgrade pip setuptools + python -m pip install -r requirements.txt - name: Cache SUNDIALS build uses: actions/cache@v3 @@ -201,6 +204,9 @@ jobs: - name: Build PETSc run: BUILD_PETSC=${{ matrix.config.build_petsc }} ./.build_petsc_for_ci.sh ${{ matrix.config.build_petsc_branch }} + - name: Build ADIOS2 + run: BUILD_ADIOS2=${{ matrix.config.build_adios2 }} ./.build_adios2_for_ci.sh + - name: Build BOUT++ run: UNIT_ONLY=${{ matrix.config.unit_only }} ./.ci_with_cmake.sh ${{ matrix.config.cmake_options }} diff --git a/.pip_install_for_ci.sh b/.pip_install_for_ci.sh deleted file mode 100755 index 4a5258cc2d..0000000000 --- a/.pip_install_for_ci.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -e - -export PATH=${HOME}/.local/bin:${PATH} -pip3 install --user --upgrade pip~=20.0 setuptools~=46.1 -pip3 install --user --upgrade scipy~=1.4 numpy~=1.18 natsort~=8.1.0 -for package in $@ -do - if test $package == "cython" - then - # fast install Cython - pip3 install --user Cython --install-option="--no-cython-compile" - elif test $package == "something_else" - then - pip3 install what_we_need - else - pip3 install --user $package - fi -done diff --git a/CMakeLists.txt b/CMakeLists.txt index 7199bb376a..ada2e6b4c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ function(bout_update_submodules) endfunction() set(BOUT_SOURCES + ./include/bout/adios_object.hxx ./include/bout/array.hxx ./include/bout/assert.hxx ./include/bout/boundary_factory.hxx @@ -146,7 +147,7 @@ set(BOUT_SOURCES ./include/bout/openmpwrap.hxx ./include/bout/operatorstencil.hxx ./include/bout/options.hxx - ./include/bout/options_netcdf.hxx + ./include/bout/options_io.hxx ./include/bout/optionsreader.hxx ./include/bout/output.hxx ./include/bout/output_bout_types.hxx @@ -325,6 +326,7 @@ set(BOUT_SOURCES ./src/solver/impls/split-rk/split-rk.cxx ./src/solver/impls/split-rk/split-rk.hxx ./src/solver/solver.cxx + ./src/sys/adios_object.cxx ./src/sys/bout_types.cxx ./src/sys/boutcomm.cxx ./src/sys/boutexception.cxx @@ -338,7 +340,11 @@ set(BOUT_SOURCES ./src/sys/options/optionparser.hxx ./src/sys/options/options_ini.cxx ./src/sys/options/options_ini.hxx + ./src/sys/options/options_io.cxx ./src/sys/options/options_netcdf.cxx + ./src/sys/options/options_netcdf.hxx + ./src/sys/options/options_adios.cxx + ./src/sys/options/options_adios.hxx ./src/sys/optionsreader.cxx ./src/sys/output.cxx ./src/sys/petsclib.cxx @@ -930,6 +936,7 @@ message(" SUNDIALS support : ${BOUT_HAS_SUNDIALS} HYPRE support : ${BOUT_HAS_HYPRE} NetCDF support : ${BOUT_HAS_NETCDF} + ADIOS support : ${BOUT_HAS_ADIOS} FFTW support : ${BOUT_HAS_FFTW} LAPACK support : ${BOUT_HAS_LAPACK} OpenMP support : ${BOUT_USE_OPENMP} diff --git a/bin/bout-config.in b/bin/bout-config.in index 697d3ddc71..a9045fff39 100755 --- a/bin/bout-config.in +++ b/bin/bout-config.in @@ -29,6 +29,7 @@ idlpath="@IDLCONFIGPATH@" pythonpath="@PYTHONCONFIGPATH@" has_netcdf="@BOUT_HAS_NETCDF@" +has_adios="@BOUT_HAS_ADIOS@" has_legacy_netcdf="@BOUT_HAS_LEGACY_NETCDF@" has_pnetcdf="@BOUT_HAS_PNETCDF@" has_pvode="@BOUT_HAS_PVODE@" @@ -71,6 +72,7 @@ Available values for OPTION include: --python Python path --has-netcdf NetCDF file support + --has-adios ADIOS file support --has-legacy-netcdf Legacy NetCDF file support --has-pnetcdf Parallel NetCDF file support --has-pvode PVODE solver support @@ -109,6 +111,7 @@ all() echo " --python -> $pythonpath" echo echo " --has-netcdf -> $has_netcdf" + echo " --has-adios -> $has_adios" echo " --has-legacy-netcdf -> $has_legacy_netcdf" echo " --has-pnetcdf -> $has_pnetcdf" echo " --has-pvode -> $has_pvode" @@ -197,6 +200,10 @@ while test $# -gt 0; do echo $has_netcdf ;; + --has-adios) + echo $has_adios + ;; + --has-legacy-netcdf) echo $has_legacy_netcdf ;; diff --git a/bout++Config.cmake.in b/bout++Config.cmake.in index e33e950e6f..3d824e455f 100644 --- a/bout++Config.cmake.in +++ b/bout++Config.cmake.in @@ -15,6 +15,7 @@ set(BOUT_USE_METRIC_3D @BOUT_USE_METRIC_3D@) set(BOUT_HAS_PVODE @BOUT_HAS_PVODE@) set(BOUT_HAS_NETCDF @BOUT_HAS_NETCDF@) +set(BOUT_HAS_ADIOS @BOUT_HAS_ADIOS@) set(BOUT_HAS_FFTW @BOUT_HAS_FFTW@) set(BOUT_HAS_LAPACK @BOUT_HAS_LAPACK@) set(BOUT_HAS_PETSC @BOUT_HAS_PETSC@) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index 55f201bdad..e1d6f00cb4 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -156,6 +156,7 @@ option(BOUT_USE_NETCDF "Enable support for NetCDF output" ON) option(BOUT_DOWNLOAD_NETCDF_CXX4 "Download and build netCDF-cxx4" OFF) if (BOUT_USE_NETCDF) if (BOUT_DOWNLOAD_NETCDF_CXX4) + message(STATUS "Downloading and configuring NetCDF-cxx4") include(FetchContent) FetchContent_Declare( netcdf-cxx4 @@ -185,6 +186,44 @@ endif() message(STATUS "NetCDF support: ${BOUT_USE_NETCDF}") set(BOUT_HAS_NETCDF ${BOUT_USE_NETCDF}) +option(BOUT_USE_ADIOS "Enable support for ADIOS output" ON) +option(BOUT_DOWNLOAD_ADIOS "Download and build ADIOS2" OFF) +if (BOUT_USE_ADIOS) + if (BOUT_DOWNLOAD_ADIOS) + message(STATUS "Downloading and configuring ADIOS2") + include(FetchContent) + FetchContent_Declare( + adios2 + GIT_REPOSITORY https://github.com/ornladios/ADIOS2.git + GIT_TAG origin/master + GIT_SHALLOW 1 + ) + set(ADIOS2_USE_MPI ON CACHE BOOL "" FORCE) + set(ADIOS2_USE_Fortran OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Python OFF CACHE BOOL "" FORCE) + set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + # Disable testing, or ADIOS will try to find or install GTEST + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + # Note: SST requires but doesn't check at configure time + set(ADIOS2_USE_SST OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(adios2) + target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi) + message(STATUS "ADIOS2 done configuring") + else() + find_package(ADIOS2) + if (ADIOS2_FOUND) + ENABLE_LANGUAGE(C) + find_package(MPI REQUIRED COMPONENTS C) + target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi MPI::MPI_C) + else() + set(BOUT_USE_ADIOS OFF) + endif() + endif() +endif() +message(STATUS "ADIOS support: ${BOUT_USE_ADIOS}") +set(BOUT_HAS_ADIOS ${BOUT_USE_ADIOS}) + + option(BOUT_USE_FFTW "Enable support for FFTW" ON) if (BOUT_USE_FFTW) find_package(FFTW REQUIRED) diff --git a/cmake_build_defines.hxx.in b/cmake_build_defines.hxx.in index a637dbc46a..ed6e8685f6 100644 --- a/cmake_build_defines.hxx.in +++ b/cmake_build_defines.hxx.in @@ -13,6 +13,7 @@ #cmakedefine01 BOUT_HAS_IDA #cmakedefine01 BOUT_HAS_LAPACK #cmakedefine01 BOUT_HAS_NETCDF +#cmakedefine01 BOUT_HAS_ADIOS #cmakedefine01 BOUT_HAS_PETSC #cmakedefine01 BOUT_HAS_PRETTY_FUNCTION #cmakedefine01 BOUT_HAS_PVODE diff --git a/include/bout/adios_object.hxx b/include/bout/adios_object.hxx new file mode 100755 index 0000000000..9d2f545b46 --- /dev/null +++ b/include/bout/adios_object.hxx @@ -0,0 +1,83 @@ +/*!************************************************************************ + * Provides access to the ADIOS library, handling initialisation and + * finalisation. + * + * Usage + * ----- + * + * #include + * + **************************************************************************/ + +#ifndef ADIOS_OBJECT_HXX +#define ADIOS_OBJECT_HXX + +#include "bout/build_config.hxx" + +#if BOUT_HAS_ADIOS + +#include +#include +#include + +namespace bout { + +void ADIOSInit(MPI_Comm comm); +void ADIOSInit(const std::string configFile, MPI_Comm comm); +void ADIOSFinalize(); + +using ADIOSPtr = std::shared_ptr; +using EnginePtr = std::shared_ptr; +using IOPtr = std::shared_ptr; + +ADIOSPtr GetADIOSPtr(); +IOPtr GetIOPtr(const std::string IOName); + +class ADIOSStream { +public: + adios2::IO io; + adios2::Engine engine; + adios2::Variable vTime; + adios2::Variable vStep; + int adiosStep = 0; + bool isInStep = false; // true if BeginStep was called and EndStep was not yet called + + /** create or return the ADIOSStream based on the target file name */ + static ADIOSStream& ADIOSGetStream(const std::string& fname); + + ~ADIOSStream(); + + template + adios2::Variable GetValueVariable(const std::string& varname) { + auto v = io.InquireVariable(varname); + if (!v) { + v = io.DefineVariable(varname); + } + return v; + } + + template + adios2::Variable GetArrayVariable(const std::string& varname, adios2::Dims& shape) { + adios2::Variable v = io.InquireVariable(varname); + if (!v) { + adios2::Dims start(shape.size()); + v = io.DefineVariable(varname, shape, start, shape); + } else { + v.SetShape(shape); + } + return v; + } + +private: + ADIOSStream(const std::string fname) : fname(fname){}; + std::string fname; +}; + +/** Set user parameters for an IO group */ +void ADIOSSetParameters(const std::string& input, const char delimKeyValue, + const char delimItem, adios2::IO& io); + +} // namespace bout + +#endif //BOUT_HAS_ADIOS +#endif //ADIOS_OBJECT_HXX diff --git a/include/bout/bout.hxx b/include/bout/bout.hxx index 5e718dde33..d929a19c2f 100644 --- a/include/bout/bout.hxx +++ b/include/bout/bout.hxx @@ -2,8 +2,6 @@ * * @mainpage BOUT++ * - * @version 3.0 - * * @par Description * Framework for the solution of partial differential * equations, in particular fluid models in plasma physics. @@ -33,8 +31,8 @@ * **************************************************************************/ -#ifndef __BOUT_H__ -#define __BOUT_H__ +#ifndef BOUT_H +#define BOUT_H #include "bout/build_config.hxx" @@ -44,7 +42,7 @@ #include "bout/field3d.hxx" #include "bout/globals.hxx" #include "bout/mesh.hxx" -#include "bout/options_netcdf.hxx" +#include "bout/options_io.hxx" #include "bout/output.hxx" #include "bout/smoothing.hxx" // Smoothing functions #include "bout/solver.hxx" @@ -206,4 +204,4 @@ private: */ int BoutFinalise(bool write_settings = true); -#endif // __BOUT_H__ +#endif // BOUT_H diff --git a/include/bout/build_config.hxx b/include/bout/build_config.hxx index a98c615c77..c97962f7cf 100644 --- a/include/bout/build_config.hxx +++ b/include/bout/build_config.hxx @@ -17,6 +17,7 @@ constexpr auto has_gettext = static_cast(BOUT_HAS_GETTEXT); constexpr auto has_lapack = static_cast(BOUT_HAS_LAPACK); constexpr auto has_legacy_netcdf = static_cast(BOUT_HAS_LEGACY_NETCDF); constexpr auto has_netcdf = static_cast(BOUT_HAS_NETCDF); +constexpr auto has_adios = static_cast(BOUT_HAS_ADIOS); constexpr auto has_petsc = static_cast(BOUT_HAS_PETSC); constexpr auto has_hypre = static_cast(BOUT_HAS_HYPRE); constexpr auto has_umpire = static_cast(BOUT_HAS_UMPIRE); diff --git a/include/bout/field_accessor.hxx b/include/bout/field_accessor.hxx index 16661a0e75..69b58da979 100644 --- a/include/bout/field_accessor.hxx +++ b/include/bout/field_accessor.hxx @@ -39,9 +39,9 @@ struct BoutRealArray { /// Cast operators, so can be assigned to a raw pointer /// Note: Not explicit, so can be cast implicitly - operator BoutReal*() { return data; } + BOUT_HOST_DEVICE operator BoutReal*() { return data; } - operator const BoutReal*() const { return data; } + BOUT_HOST_DEVICE operator const BoutReal*() const { return data; } }; /// Thin wrapper around field data, for fast but unsafe access diff --git a/include/bout/generic_factory.hxx b/include/bout/generic_factory.hxx index 3a2a63c94c..f7a0692af8 100644 --- a/include/bout/generic_factory.hxx +++ b/include/bout/generic_factory.hxx @@ -1,8 +1,8 @@ /// Base type for factories #pragma once -#ifndef __BOUT_GENERIC_FACTORY_H__ -#define __BOUT_GENERIC_FACTORY_H__ +#ifndef BOUT_GENERIC_FACTORY_H +#define BOUT_GENERIC_FACTORY_H #include "bout/boutexception.hxx" #include "bout/options.hxx" @@ -259,4 +259,4 @@ public: }; }; -#endif // __BOUT_GENERIC_FACTORY_H__ +#endif // BOUT_GENERIC_FACTORY_H diff --git a/include/bout/mesh.hxx b/include/bout/mesh.hxx index 45ea392482..2a336383f4 100644 --- a/include/bout/mesh.hxx +++ b/include/bout/mesh.hxx @@ -503,9 +503,20 @@ public: int GlobalNx, GlobalNy, GlobalNz; ///< Size of the global arrays. Note: can have holes /// Size of the global arrays excluding boundary points. int GlobalNxNoBoundaries, GlobalNyNoBoundaries, GlobalNzNoBoundaries; + + /// Note: These offsets only correct if Y guards are not included in the global array + /// and are corrected in gridfromfile.cxx int OffsetX, OffsetY, OffsetZ; ///< Offset of this mesh within the global array ///< so startx on this processor is OffsetX in global + /// Map between local and global indices + /// (MapGlobalX, MapGlobalY, MapGlobalZ) in the global index space maps to (MapLocalX, MapLocalY, MapLocalZ) locally. + /// Note that boundary cells are included in the global index space, but communication + /// guard cells are not. + int MapGlobalX, MapGlobalY, MapGlobalZ; ///< Start global indices + int MapLocalX, MapLocalY, MapLocalZ; ///< Start local indices + int MapCountX, MapCountY, MapCountZ; ///< Size of the mapped region + /// Returns the number of unique cells (i.e., ones not used for /// communication) on this processor for 3D fields. Boundaries /// are only included to a depth of 1. diff --git a/include/bout/options_io.hxx b/include/bout/options_io.hxx new file mode 100644 index 0000000000..65c3cb7dd1 --- /dev/null +++ b/include/bout/options_io.hxx @@ -0,0 +1,178 @@ +/// Parent class for IO to binary files and streams +/// +/// +/// Usage: +/// +/// 1. Dump files, containing time history: +/// +/// auto dump = OptionsIOFactory::getInstance().createOutput(); +/// dump->write(data); +/// +/// where data is an Options tree. By default dump files are configured +/// with the root `output` section, or an Option tree can be passed to +/// `createOutput`. +/// +/// 2. Restart files: +/// +/// auto restart = OptionsIOFactory::getInstance().createOutput(); +/// restart->write(data); +/// +/// where data is an Options tree. By default restart files are configured +/// with the root `restart_files` section, or an Option tree can be passed to +/// `createRestart`. +/// +/// 3. Ad-hoc single files +/// Note: The caller should consider how multiple processors interact with the file. +/// +/// auto file = OptionsIOFactory::getInstance().createFile("some_file.nc"); +/// or +/// auto file = OptionsIO::create("some_file.nc"); +/// +/// + +#pragma once + +#ifndef OPTIONS_IO_H +#define OPTIONS_IO_H + +#include "bout/build_config.hxx" +#include "bout/generic_factory.hxx" +#include "bout/options.hxx" + +#include +#include + +namespace bout { + +class OptionsIO { +public: + /// No default constructor, as settings are required + OptionsIO() = delete; + + /// Constructor specifies the kind of file, and options to control + /// the name of file, mode of operation etc. + OptionsIO(Options&) {} + + virtual ~OptionsIO() = default; + + OptionsIO(const OptionsIO&) = delete; + OptionsIO(OptionsIO&&) noexcept = default; + OptionsIO& operator=(const OptionsIO&) = delete; + OptionsIO& operator=(OptionsIO&&) noexcept = default; + + /// Read options from file + virtual Options read() = 0; + + /// Write options to file + void write(const Options& options) { write(options, "t"); } + virtual void write(const Options& options, const std::string& time_dim) = 0; + + /// NetCDF: Check that all variables with the same time dimension have the + /// same size in that dimension. Throws BoutException if there are + /// any differences, otherwise is silent. + /// ADIOS: Indicate completion of an output step. + virtual void verifyTimesteps() const = 0; + + /// Create an OptionsIO for I/O to the given file. + /// This uses the default file type and default options. + static std::unique_ptr create(const std::string& file); + + /// Create an OptionsIO for I/O to the given file. + /// The file will be configured using the given `config` options: + /// - "type" : string The file type e.g. "netcdf" or "adios" + /// - "file" : string Name of the file + /// - "append" : bool Append to existing data (Default is false) + static std::unique_ptr create(Options& config); + + /// Create an OptionsIO for I/O to the given file. + /// The file will be configured using the given `config` options: + /// - "type" : string The file type e.g. "netcdf" or "adios" + /// - "file" : string Name of the file + /// - "append" : bool Append to existing data (Default is false) + /// + /// Example: + /// + /// auto file = OptionsIO::create({ + /// {"file", "some_file.nc"}, + /// {"type", "netcdf"}, + /// {"append", false} + /// }); + static std::unique_ptr + create(std::initializer_list> config_list) { + Options config(config_list); // Construct an Options to pass by reference + return create(config); + } +}; + +class OptionsIOFactory : public Factory { +public: + static constexpr auto type_name = "OptionsIO"; + static constexpr auto section_name = "io"; + static constexpr auto option_name = "type"; + static constexpr auto default_type = +#if BOUT_HAS_NETCDF + "netcdf"; +#elif BOUT_HAS_ADIOS + "adios"; +#else + "invalid"; +#endif + + /// Create a restart file, configured with options (if given), + /// or root "restart_files" section. + /// + /// Options: + /// - "type" The type of file e.g "netcdf" or "adios" + /// - "file" Name of the file. Default is /.[type-dependent] + /// - "path" Path to restart files. Default is root "datadir" option, + /// that defaults to "data" + /// - "prefix" Default is "BOUT.restart" + ReturnType createRestart(Options* optionsptr = nullptr) const; + + /// Create an output file for writing time history. + /// Configure with options (if given), or root "output" section. + /// + /// Options: + /// - "type" The type of file e.g "netcdf" or "adios" + /// - "file" Name of the file. Default is /.[type] + /// - "path" Path to output files. Default is root "datadir" option, + /// that defaults to "data" + /// - "prefix" Default is "BOUT.dmp" + /// - "append" Append to existing file? Default is root "append" option, + /// that defaults to false. + ReturnType createOutput(Options* optionsptr = nullptr) const; + + /// Create a single file (e.g. mesh file) of the default type + ReturnType createFile(const std::string& file) const; +}; + +/// Simpler name for Factory registration helper class +/// +/// Usage: +/// +/// #include +/// namespace { +/// RegisterOptionsIO registeroptionsiomine("myoptionsio"); +/// } +template +using RegisterOptionsIO = OptionsIOFactory::RegisterInFactory; + +/// Simpler name for indicating that an OptionsIO implementation +/// is unavailable. +/// +/// Usage: +/// +/// namespace { +/// RegisterUnavailableOptionsIO +/// unavailablemyoptionsio("myoptiosio", "BOUT++ was not configured with MyOptionsIO"); +/// } +using RegisterUnavailableOptionsIO = OptionsIOFactory::RegisterUnavailableInFactory; + +/// Convenient wrapper function around OptionsIOFactory::createOutput +/// Opens a dump file configured with the `output` root section, +/// and writes the given `data` to the file. +void writeDefaultOutputFile(Options& data); + +} // namespace bout + +#endif // OPTIONS_IO_H diff --git a/include/bout/options_netcdf.hxx b/include/bout/options_netcdf.hxx deleted file mode 100644 index 2fdb71c6d4..0000000000 --- a/include/bout/options_netcdf.hxx +++ /dev/null @@ -1,118 +0,0 @@ - -#pragma once - -#ifndef __OPTIONS_NETCDF_H__ -#define __OPTIONS_NETCDF_H__ - -#include "bout/build_config.hxx" - -#if !BOUT_HAS_NETCDF || BOUT_HAS_LEGACY_NETCDF - -#include - -#include "bout/boutexception.hxx" -#include "bout/options.hxx" - -namespace bout { - -class OptionsNetCDF { -public: - enum class FileMode { - replace, ///< Overwrite file when writing - append ///< Append to file when writing - }; - - OptionsNetCDF(const std::string& filename, FileMode mode = FileMode::replace) {} - OptionsNetCDF(const OptionsNetCDF&) = default; - OptionsNetCDF(OptionsNetCDF&&) = default; - OptionsNetCDF& operator=(const OptionsNetCDF&) = default; - OptionsNetCDF& operator=(OptionsNetCDF&&) = default; - - /// Read options from file - Options read() { throw BoutException("OptionsNetCDF not available\n"); } - - /// Write options to file - void write(const Options& options) { - throw BoutException("OptionsNetCDF not available\n"); - } -}; - -} // namespace bout - -#else - -#include -#include - -#include "bout/options.hxx" - -/// Forward declare netCDF file type so we don't need to depend -/// directly on netCDF -namespace netCDF { -class NcFile; -} - -namespace bout { - -class OptionsNetCDF { -public: - enum class FileMode { - replace, ///< Overwrite file when writing - append ///< Append to file when writing - }; - - // Constructors need to be defined in implementation due to forward - // declaration of NcFile - OptionsNetCDF(); - explicit OptionsNetCDF(std::string filename, FileMode mode = FileMode::replace); - ~OptionsNetCDF(); - OptionsNetCDF(const OptionsNetCDF&) = delete; - OptionsNetCDF(OptionsNetCDF&&) noexcept; - OptionsNetCDF& operator=(const OptionsNetCDF&) = delete; - OptionsNetCDF& operator=(OptionsNetCDF&&) noexcept; - - /// Read options from file - Options read(); - - /// Write options to file - void write(const Options& options) { write(options, "t"); } - void write(const Options& options, const std::string& time_dim); - - /// Check that all variables with the same time dimension have the - /// same size in that dimension. Throws BoutException if there are - /// any differences, otherwise is silent - void verifyTimesteps() const; - -private: - /// Name of the file on disk - std::string filename; - /// How to open the file for writing - FileMode file_mode{FileMode::replace}; - /// Pointer to netCDF file so we don't introduce direct dependence - std::unique_ptr data_file; -}; - -} // namespace bout - -#endif - -namespace bout { -/// Name of the directory for restart files -std::string getRestartDirectoryName(Options& options); -/// Name of the restart file on this rank -std::string getRestartFilename(Options& options); -/// Name of the restart file on \p rank -std::string getRestartFilename(Options& options, int rank); -/// Name of the main output file on this rank -std::string getOutputFilename(Options& options); -/// Name of the main output file on \p rank -std::string getOutputFilename(Options& options, int rank); -/// Write `Options::root()` to the main output file, overwriting any -/// existing files -void writeDefaultOutputFile(); -/// Write \p options to the main output file, overwriting any existing -/// files -void writeDefaultOutputFile(Options& options); -} // namespace bout - -#endif // __OPTIONS_NETCDF_H__ diff --git a/include/bout/physicsmodel.hxx b/include/bout/physicsmodel.hxx index a9a7d7344d..e0f046eb1f 100644 --- a/include/bout/physicsmodel.hxx +++ b/include/bout/physicsmodel.hxx @@ -42,7 +42,7 @@ class PhysicsModel; #include "bout/macro_for_each.hxx" #include "bout/msg_stack.hxx" #include "bout/options.hxx" -#include "bout/options_netcdf.hxx" +#include "bout/options_io.hxx" #include "bout/sys/variant.hxx" #include "bout/unused.hxx" #include "bout/utils.hxx" @@ -88,9 +88,6 @@ public: void add(Vector2D* value, const std::string& name, bool save_repeat = false); void add(Vector3D* value, const std::string& name, bool save_repeat = false); - /// Write stored data to file immediately - bool write(); - private: /// Helper struct to save enough information so that we can save an /// object to file later @@ -148,7 +145,7 @@ public: bout::DataFileFacade restart{}; /*! - * Initialse the model, calling the init() and postInit() methods + * Initialise the model, calling the init() and postInit() methods * * Note: this is usually only called by the Solver */ @@ -383,13 +380,13 @@ private: /// State for outputs Options output_options; /// File to write the outputs to - bout::OptionsNetCDF output_file; + std::unique_ptr output_file; /// Should we write output files bool output_enabled{true}; /// Stores the state for restarting Options restart_options; /// File to write the restart-state to - bout::OptionsNetCDF restart_file; + std::unique_ptr restart_file; /// Should we write restart files bool restart_enabled{true}; /// Split operator model? diff --git a/include/bout/solver.hxx b/include/bout/solver.hxx index 8a3f07c27a..ef7cbe63eb 100644 --- a/include/bout/solver.hxx +++ b/include/bout/solver.hxx @@ -33,8 +33,8 @@ * **************************************************************************/ -#ifndef __SOLVER_H__ -#define __SOLVER_H__ +#ifndef SOLVER_H +#define SOLVER_H #include "bout/build_config.hxx" @@ -597,4 +597,4 @@ private: BoutReal output_timestep; }; -#endif // __SOLVER_H__ +#endif // SOLVER_H diff --git a/manual/sphinx/index.rst b/manual/sphinx/index.rst index 46728c7119..9f661ca187 100644 --- a/manual/sphinx/index.rst +++ b/manual/sphinx/index.rst @@ -42,6 +42,7 @@ The documentation is divided into the following sections: user_docs/boundary_options user_docs/testing user_docs/gpu_support + user_docs/adios2 .. toctree:: :maxdepth: 2 diff --git a/manual/sphinx/user_docs/adios2.rst b/manual/sphinx/user_docs/adios2.rst new file mode 100644 index 0000000000..8a6228cd3a --- /dev/null +++ b/manual/sphinx/user_docs/adios2.rst @@ -0,0 +1,45 @@ +.. _sec-adios2: + +ADIOS2 support +============== + +This section summarises the use of `ADIOS2 `_ in BOUT++. + +Installation +------------ + +The easiest way to configure BOUT++ with ADIOS2 is to tell CMake to download and build it +with this flag:: + + -DBOUT_DOWNLOAD_ADIOS=ON + +The ``master`` branch will be downloaded from `Github `_, +configured and built with BOUT++. + +Alternatively, if ADIOS is already installed then the following flags can be used:: + + -DBOUT_USE_ADIOS=ON -DADIOS2_ROOT=/path/to/adios2 + +Output files +------------ + +The output (dump) files are controlled with the root ``output`` options. +By default the output format is NetCDF, so to use ADIOS2 instead set +the output type in BOUT.inp:: + + [output] + type = adios + +or on the BOUT++ command line set ``output:type=adios``. The default +prefix is "BOUT.dmp" so the ADIOS file will be called "BOUT.dmp.bp". To change this, +set the ``output:prefix`` option. + +Restart files +------------- + +The restart files are contolled with the root ``restart_files`` options, +so to read and write restarts from an ADIOS dataset, put in BOUT.inp:: + + [restart_files] + type = adios + diff --git a/manual/sphinx/user_docs/advanced_install.rst b/manual/sphinx/user_docs/advanced_install.rst index 957173b820..e25be12b4b 100644 --- a/manual/sphinx/user_docs/advanced_install.rst +++ b/manual/sphinx/user_docs/advanced_install.rst @@ -170,8 +170,10 @@ for a production run use: File formats ------------ -BOUT++ can currently use the NetCDF-4_ file format, with experimental -support for the parallel flavour. NetCDF is a widely used format and +BOUT++ can currently use the NetCDF-4_ file format and the ADIOS2 library +for high-performance parallel output. + +NetCDF is a widely used format and has many tools for viewing and manipulating files. .. _NetCDF-4: https://www.unidata.ucar.edu/software/netcdf/ diff --git a/manual/sphinx/user_docs/bout_options.rst b/manual/sphinx/user_docs/bout_options.rst index 5422558ff9..976faa0544 100644 --- a/manual/sphinx/user_docs/bout_options.rst +++ b/manual/sphinx/user_docs/bout_options.rst @@ -523,6 +523,12 @@ options available are listed in table :numref:`tab-outputopts`. +-------------+----------------------------------------------------+--------------+ | enabled | Writing is enabled | true | +-------------+----------------------------------------------------+--------------+ + | type | File type e.g. "netcdf" or "adios" | "netcdf" | + +-------------+----------------------------------------------------+--------------+ + | prefix | File name prefix | "BOUT.dmp" | + +-------------+----------------------------------------------------+--------------+ + | path | Directory to write the file into | ``datadir`` | + +-------------+----------------------------------------------------+--------------+ | floats | Write floats rather than doubles | false | +-------------+----------------------------------------------------+--------------+ | flush | Flush the file to disk after each write | true | @@ -531,8 +537,6 @@ options available are listed in table :numref:`tab-outputopts`. +-------------+----------------------------------------------------+--------------+ | openclose | Re-open the file for each write, and close after | true | +-------------+----------------------------------------------------+--------------+ - | parallel | Use parallel I/O | false | - +-------------+----------------------------------------------------+--------------+ | @@ -541,20 +545,6 @@ want to exclude I/O from the timings. **floats** can be used to reduce the size of the output files: files are stored as double by default, but setting **floats = true** changes the output to single-precision floats. -To enable parallel I/O for either output or restart files, set - -.. code-block:: cfg - - parallel = true - -in the output or restart section. If you have compiled BOUT++ with a -parallel I/O library such as pnetcdf (see -:ref:`sec-advancedinstall`), then rather than outputting one file per -processor, all processors will output to the same file. For restart -files this is particularly useful, as it means that you can restart a -job with a different number of processors. Note that this feature is -still experimental, and incomplete: output dump files are not yet -supported by the collect routines. Implementation -------------- @@ -833,30 +823,30 @@ This is currently quite rudimentary and needs improving. .. _sec-options-netcdf: -Reading and writing to NetCDF ------------------------------ +Reading and writing to binary formats +------------------------------------- -The `bout::OptionsNetCDF` class provides an interface to read and -write options. Examples are in integrated test +The `bout::OptionsIO` class provides an interface to read and +write options to binary files. Examples are in integrated test ``tests/integrated/test-options-netcdf/`` To write the current `Options` tree (e.g. from ``BOUT.inp``) to a NetCDF file:: - bout::OptionsNetCDF("settings.nc").write(Options::root()); + bout::OptionsIO::create("settings.nc")->write(Options::root()); and to read it in again:: - Options data = bout::OptionsNetCDF("settings.nc").read(); + Options data = bout::OptionsIO::create("settings.nc")->read(); Fields can also be stored and written:: Options fields; fields["f2d"] = Field2D(1.0); fields["f3d"] = Field3D(2.0); - bout::OptionsNetCDF("fields.nc").write(fields); + bout::OptionsIO::create("fields.nc").write(fields); -This should allow the input settings and evolving variables to be +This allows the input settings and evolving variables to be combined into a single tree (see above on joining trees) and written to the output dump or restart files. @@ -865,7 +855,7 @@ an ``Array``, 2D as ``Matrix`` and 3D as ``Tensor``. These can be extracted directly from the ``Options`` tree, or converted to a Field:: - Options fields_in = bout::OptionsNetCDF("fields.nc").read(); + Options fields_in = bout::OptionsIO::create("fields.nc")->read(); Field2D f2d = fields_in["f2d"].as(); Field3D f3d = fields_in["f3d"].as(); @@ -907,7 +897,7 @@ automatically set the ``"time_dimension"`` attribute:: // Or use `assignRepeat` to do it automatically: data["field"].assignRepeat(Field3D(2.0)); - bout::OptionsNetCDF("time.nc").write(data); + bout::OptionsIO::create("time.nc")->write(data); // Update time-dependent values. This can be done without `force` if the time_dimension // attribute is set @@ -915,13 +905,13 @@ automatically set the ``"time_dimension"`` attribute:: data["field"] = Field3D(3.0); // Append data to file - bout::OptionsNetCDF("time.nc", bout::OptionsNetCDF::FileMode::append).write(data); + bout::OptionsIO({{"file", "time.nc"}, {"append", true}})->write(data); -.. note:: By default, `bout::OptionsNetCDF::write` will only write variables +.. note:: By default, `bout::OptionsIO::write` will only write variables with a ``"time_dimension"`` of ``"t"``. You can write variables with a different time dimension by passing it as the second argument: - ``OptionsNetCDF(filename).write(options, "t2")`` for example. + ``OptionsIO::create(filename)->write(options, "t2")`` for example. FFT diff --git a/manual/sphinx/user_docs/installing.rst b/manual/sphinx/user_docs/installing.rst index fc1b2ce2da..eb155909bf 100644 --- a/manual/sphinx/user_docs/installing.rst +++ b/manual/sphinx/user_docs/installing.rst @@ -373,6 +373,10 @@ For SUNDIALS, use ``-DBOUT_DOWNLOAD_SUNDIALS=ON``. If using ``ccmake`` this opti may not appear initially. This automatically sets ``BOUT_USE_SUNDIALS=ON``, and configures SUNDIALS to use MPI. +For ADIOS2, use ``-DBOUT_DOWNLOAD_ADIOS=ON``. This will download and +configure `ADIOS2 `_, enabling BOUT++ +to read and write this high-performance parallel file format. + Bundled Dependencies ~~~~~~~~~~~~~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index 03f43a92d2..75358b10db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ Jinja2>=2.11.3 numpy>=1.14.1 scipy>=1.0.0 -netcdf4>=1.3.1 +netcdf4~=1.6.0 matplotlib>=2.0.0 -Cython>=0.29.0 +Cython~=3.0.0 +boututils~=0.2.1 +boutdata~=0.2.1 diff --git a/src/bout++.cxx b/src/bout++.cxx index 127cb6ff4a..1dc93dc76d 100644 --- a/src/bout++.cxx +++ b/src/bout++.cxx @@ -4,9 +4,9 @@ * Adapted from the BOUT code by B.Dudson, University of York, Oct 2007 * ************************************************************************** - * Copyright 2010 B.D.Dudson, S.Farley, M.V.Umansky, X.Q.Xu + * Copyright 2010-2023 BOUT++ contributors * - * Contact Ben Dudson, bd512@york.ac.uk + * Contact Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -59,6 +59,10 @@ const char DEFAULT_DIR[] = "data"; #include "bout/bout.hxx" #undef BOUT_NO_USING_NAMESPACE_BOUTGLOBALS +#if BOUT_HAS_ADIOS +#include "bout/adios_object.hxx" +#endif + #include #include @@ -161,6 +165,10 @@ int BoutInitialise(int& argc, char**& argv) { savePIDtoFile(args.data_dir, MYPE); +#if BOUT_HAS_ADIOS + bout::ADIOSInit(BoutComm::get()); +#endif + // Print the different parts of the startup info printStartupHeader(MYPE, BoutComm::size()); printCompileTimeOptions(); @@ -565,6 +573,7 @@ void printCompileTimeOptions() { constexpr auto netcdf_flavour = has_netcdf ? (has_legacy_netcdf ? " (Legacy)" : " (NetCDF4)") : ""; output_info.write(_("\tNetCDF support {}{}\n"), is_enabled(has_netcdf), netcdf_flavour); + output_info.write(_("\tADIOS support {}\n"), is_enabled(has_adios)); output_info.write(_("\tPETSc support {}\n"), is_enabled(has_petsc)); output_info.write(_("\tPretty function name support {}\n"), is_enabled(has_pretty_function)); @@ -693,6 +702,7 @@ void addBuildFlagsToOptions(Options& options) { options["has_gettext"].force(bout::build::has_gettext); options["has_lapack"].force(bout::build::has_lapack); options["has_netcdf"].force(bout::build::has_netcdf); + options["has_adios"].force(bout::build::has_adios); options["has_petsc"].force(bout::build::has_petsc); options["has_hypre"].force(bout::build::has_hypre); options["has_umpire"].force(bout::build::has_umpire); @@ -788,6 +798,10 @@ int BoutFinalise(bool write_settings) { // Call HYPER_Finalize if not already called bout::HypreLib::cleanup(); +#if BOUT_HAS_ADIOS + bout::ADIOSFinalize(); +#endif + // MPI communicator, including MPI_Finalize() BoutComm::cleanup(); diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index f99a8a8020..c74e184be3 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -39,7 +39,8 @@ #include #include #include -#include + +#include LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Solver* solver) @@ -146,14 +147,12 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, } // FIXME: This needs to be converted to outputVars - if (solver == nullptr or dump == nullptr) { - output_warn << "Warning: Need to pass both a Solver and a Datafile to " + if (solver == nullptr) { + output_warn << "Warning: Need to pass a Solver to " "Laplacian::create() to get iteration counts in the output." << endl; } else { solver->addMonitor(&monitor); - auto name = opt->name(); - dump->addRepeat(average_iterations, name + "_average_iterations"); } } @@ -182,41 +181,41 @@ Field3D LaplaceHypre3d::solve(const Field3D& b_in, const Field3D& x0) { // boundary cells are finite BOUT_FOR_SERIAL(i, indexer->getRegionInnerX()) { const BoutReal val = (inner_boundary_flags & INVERT_SET) ? x0[i] : 0.; - ASSERT1(finite(val)); + ASSERT1(std::isfinite(val)); if (!(inner_boundary_flags & INVERT_RHS)) { b[i] = val; } else { - ASSERT1(finite(b[i])); + ASSERT1(std::isfinite(b[i])); } } BOUT_FOR_SERIAL(i, indexer->getRegionOuterX()) { const BoutReal val = (outer_boundary_flags & INVERT_SET) ? x0[i] : 0.; - ASSERT1(finite(val)); + ASSERT1(std::isfinite(val)); if (!(outer_boundary_flags & INVERT_RHS)) { b[i] = val; } else { - ASSERT1(finite(b[i])); + ASSERT1(std::isfinite(b[i])); } } BOUT_FOR_SERIAL(i, indexer->getRegionLowerY()) { const BoutReal val = (lower_boundary_flags & INVERT_SET) ? x0[i] : 0.; - ASSERT1(finite(val)); + ASSERT1(std::isfinite(val)); if (!(lower_boundary_flags & INVERT_RHS)) { b[i] = val; } else { - ASSERT1(finite(b[i])); + ASSERT1(std::isfinite(b[i])); } } BOUT_FOR_SERIAL(i, indexer->getRegionUpperY()) { const BoutReal val = (upper_boundary_flags & INVERT_SET) ? x0[i] : 0.; - ASSERT1(finite(val)); + ASSERT1(std::isfinite(val)); if (!(upper_boundary_flags & INVERT_RHS)) { b[i] = val; } else { - ASSERT1(finite(b[i])); + ASSERT1(std::isfinite(b[i])); } } CALI_MARK_END("LaplaceHypre3d_solve:AdjustBoundary"); diff --git a/src/mesh/data/gridfromfile.cxx b/src/mesh/data/gridfromfile.cxx index eed70776b3..2e23ce8eb8 100644 --- a/src/mesh/data/gridfromfile.cxx +++ b/src/mesh/data/gridfromfile.cxx @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -14,7 +14,7 @@ #include GridFile::GridFile(std::string gridfilename) - : GridDataSource(true), data(bout::OptionsNetCDF(gridfilename).read()), + : GridDataSource(true), data(bout::OptionsIO::create(gridfilename)->read()), filename(std::move(gridfilename)) { TRACE("GridFile constructor"); diff --git a/src/mesh/impls/bout/boutmesh.cxx b/src/mesh/impls/bout/boutmesh.cxx index c2c9b51ba3..9f9780c70f 100644 --- a/src/mesh/impls/bout/boutmesh.cxx +++ b/src/mesh/impls/bout/boutmesh.cxx @@ -333,7 +333,9 @@ void BoutMesh::setDerivedGridSizes() { } GlobalNx = nx; - GlobalNy = ny + 2 * MYG; + GlobalNy = + ny + + 2 * MYG; // Note: For double null this should be be 4 * MYG if boundary cells are stored GlobalNz = nz; // If we've got a second pair of diverator legs, we need an extra @@ -374,6 +376,7 @@ void BoutMesh::setDerivedGridSizes() { } // Set global offsets + // Note: These don't properly include guard/boundary cells OffsetX = PE_XIND * MXSUB; OffsetY = PE_YIND * MYSUB; OffsetZ = 0; @@ -392,6 +395,48 @@ void BoutMesh::setDerivedGridSizes() { zstart = MZG; zend = MZG + MZSUB - 1; + + // Mapping local to global indices + if (periodicX) { + // No boundary cells in X + MapGlobalX = PE_XIND * MXSUB; + MapLocalX = MXG; + MapCountX = MXSUB; + } else { + // X boundaries stored for firstX and lastX processors + if (firstX()) { + MapGlobalX = 0; + MapLocalX = 0; + MapCountX = MXG + MXSUB; + } else { + MapGlobalX = MXG + PE_XIND * MXSUB; + MapLocalX = MXG; // Guard cells not included + MapCountX = MXSUB; + } + if (lastX()) { + // Doesn't change the origin, but adds outer X boundary cells + MapCountX += MXG; + } + } + + if (PE_YIND == 0) { + // Include Y boundary cells + MapGlobalY = 0; + MapLocalY = 0; + MapCountY = MYG + MYSUB; + } else { + MapGlobalY = MYG + PE_YIND * MYSUB; + MapLocalY = MYG; + MapCountY = MYSUB; + } + if (PE_YIND == NYPE - 1) { + // Include Y upper boundary region. + MapCountY += MYG; + } + + MapGlobalZ = 0; + MapLocalZ = MZG; // Omit boundary cells + MapCountZ = MZSUB; } int BoutMesh::load() { diff --git a/src/physics/physicsmodel.cxx b/src/physics/physicsmodel.cxx index cac4bda5cc..9f538895ed 100644 --- a/src/physics/physicsmodel.cxx +++ b/src/physics/physicsmodel.cxx @@ -58,35 +58,25 @@ void DataFileFacade::add(Vector3D* value, const std::string& name, bool save_rep add(value->y, name_prefix + "y"s, save_repeat); add(value->z, name_prefix + "z"s, save_repeat); } - -bool DataFileFacade::write() { - for (const auto& item : data) { - bout::utils::visit(bout::OptionsConversionVisitor{Options::root(), item.name}, - item.value); - if (item.repeat) { - Options::root()[item.name].attributes["time_dimension"] = "t"; - } - } - writeDefaultOutputFile(); - return true; -} } // namespace bout PhysicsModel::PhysicsModel() - : mesh(bout::globals::mesh), - output_file(bout::getOutputFilename(Options::root()), - Options::root()["append"] - .doc("Add output data to existing (dump) files?") - .withDefault(false) - ? bout::OptionsNetCDF::FileMode::append - : bout::OptionsNetCDF::FileMode::replace), - output_enabled(Options::root()["output"]["enabled"] - .doc("Write output files") - .withDefault(true)), - restart_file(bout::getRestartFilename(Options::root())), + : mesh(bout::globals::mesh), output_enabled(Options::root()["output"]["enabled"] + .doc("Write output files") + .withDefault(true)), restart_enabled(Options::root()["restart_files"]["enabled"] .doc("Write restart files") - .withDefault(true)) {} + .withDefault(true)) + +{ + if (output_enabled) { + output_file = bout::OptionsIOFactory::getInstance().createOutput(); + } + + if (restart_enabled) { + restart_file = bout::OptionsIOFactory::getInstance().createRestart(); + } +} void PhysicsModel::initialise(Solver* s) { if (initialised) { @@ -104,7 +94,7 @@ void PhysicsModel::initialise(Solver* s) { const bool restarting = Options::root()["restart"].withDefault(false); if (restarting) { - restart_options = restart_file.read(); + restart_options = restart_file->read(); } // Call user init code to specify evolving variables @@ -187,7 +177,7 @@ int PhysicsModel::postInit(bool restarting) { restart_options["BOUT_VERSION"].force(bout::version::as_double, "PhysicsModel"); // Write _everything_ to restart file - restart_file.write(restart_options); + restart_file->write(restart_options); } // Add monitor to the solver which calls restart.write() and @@ -219,7 +209,7 @@ void PhysicsModel::restartVars(Options& options) { void PhysicsModel::writeRestartFile() { if (restart_enabled) { - restart_file.write(restart_options); + restart_file->write(restart_options); } } @@ -227,20 +217,20 @@ void PhysicsModel::writeOutputFile() { writeOutputFile(output_options); } void PhysicsModel::writeOutputFile(const Options& options) { if (output_enabled) { - output_file.write(options, "t"); + output_file->write(options, "t"); } } void PhysicsModel::writeOutputFile(const Options& options, const std::string& time_dimension) { if (output_enabled) { - output_file.write(options, time_dimension); + output_file->write(options, time_dimension); } } void PhysicsModel::finishOutputTimestep() const { if (output_enabled) { - output_file.verifyTimesteps(); + output_file->verifyTimesteps(); } } diff --git a/src/sys/adios_object.cxx b/src/sys/adios_object.cxx new file mode 100644 index 0000000000..c7d6dab9aa --- /dev/null +++ b/src/sys/adios_object.cxx @@ -0,0 +1,98 @@ +#include "bout/build_config.hxx" + +#if BOUT_HAS_ADIOS + +#include "bout/adios_object.hxx" +#include "bout/boutexception.hxx" + +#include +#include +#include + +namespace bout { + +static ADIOSPtr adios = nullptr; +static std::unordered_map adiosStreams; + +void ADIOSInit(MPI_Comm comm) { adios = std::make_shared(comm); } + +void ADIOSInit(const std::string configFile, MPI_Comm comm) { + adios = std::make_shared(configFile, comm); +} + +void ADIOSFinalize() { + if (adios == nullptr) { + throw BoutException( + "ADIOS needs to be initialized first before calling ADIOSFinalize()"); + } + adiosStreams.clear(); + adios.reset(); +} + +ADIOSPtr GetADIOSPtr() { + if (adios == nullptr) { + throw BoutException( + "ADIOS needs to be initialized first before calling GetADIOSPtr()"); + } + return adios; +} + +IOPtr GetIOPtr(const std::string IOName) { + auto adios = GetADIOSPtr(); + IOPtr io = nullptr; + try { + io = std::make_shared(adios->AtIO(IOName)); + } catch (std::invalid_argument& e) { + } + return io; +} + +ADIOSStream::~ADIOSStream() { + if (engine) { + if (isInStep) { + engine.EndStep(); + isInStep = false; + } + engine.Close(); + } +} + +ADIOSStream& ADIOSStream::ADIOSGetStream(const std::string& fname) { + auto it = adiosStreams.find(fname); + if (it == adiosStreams.end()) { + it = adiosStreams.emplace(fname, ADIOSStream(fname)).first; + } + return it->second; +} + +void ADIOSSetParameters(const std::string& input, const char delimKeyValue, + const char delimItem, adios2::IO& io) { + auto lf_Trim = [](std::string& input) { + input.erase(0, input.find_first_not_of(" \n\r\t")); // prefixing spaces + input.erase(input.find_last_not_of(" \n\r\t") + 1); // suffixing spaces + }; + + std::istringstream inputSS(input); + std::string parameter; + while (std::getline(inputSS, parameter, delimItem)) { + const size_t position = parameter.find(delimKeyValue); + if (position == parameter.npos) { + throw BoutException("ADIOSSetParameters(): wrong format for IO parameter " + + parameter + ", format must be key" + delimKeyValue + + "value for each entry"); + } + + std::string key = parameter.substr(0, position); + lf_Trim(key); + std::string value = parameter.substr(position + 1); + lf_Trim(value); + if (value.length() == 0) { + throw BoutException("ADIOS2SetParameters: empty value in IO parameter " + parameter + + ", format must be key" + delimKeyValue + "value"); + } + io.SetParameter(key, value); + } +} + +} // namespace bout +#endif //BOUT_HAS_ADIOS diff --git a/src/sys/options/options_adios.cxx b/src/sys/options/options_adios.cxx new file mode 100644 index 0000000000..e93a1fea94 --- /dev/null +++ b/src/sys/options/options_adios.cxx @@ -0,0 +1,548 @@ +#include "bout/build_config.hxx" + +#if BOUT_HAS_ADIOS + +#include "options_adios.hxx" +#include "bout/adios_object.hxx" + +#include "bout/bout.hxx" +#include "bout/boutexception.hxx" +#include "bout/globals.hxx" +#include "bout/mesh.hxx" +#include "bout/sys/timer.hxx" + +#include "adios2.h" +#include +#include +#include + +namespace bout { +/// Name of the attribute used to track individual variable's time indices +constexpr auto current_time_index_name = "current_time_index"; + +OptionsADIOS::OptionsADIOS(Options& options) : OptionsIO(options) { + if (options["file"].doc("File name. Defaults to /.pb").isSet()) { + filename = options["file"].as(); + } else { + // Both path and prefix must be set + filename = fmt::format("{}/{}.bp", options["path"].as(), + options["prefix"].as()); + } + + file_mode = (options["append"].doc("Append to existing file?").withDefault(false)) + ? FileMode::append + : FileMode::replace; + + singleWriteFile = options["singleWriteFile"].withDefault(false); +} + +template +Options readVariable(adios2::Engine& reader, adios2::IO& io, const std::string& name, + const std::string& type) { + std::vector data; + adios2::Variable variable = io.InquireVariable(name); + + if (variable.ShapeID() == adios2::ShapeID::GlobalValue) { + T value; + reader.Get(variable, &value, adios2::Mode::Sync); + return Options(value); + } + + if (variable.ShapeID() == adios2::ShapeID::LocalArray) { + throw std::invalid_argument( + "ADIOS reader did not implement reading local arrays like " + type + " " + name + + " in file " + reader.Name()); + } + + if (type != "double" && type != "float") { + throw std::invalid_argument( + "ADIOS reader did not implement reading arrays that are not double/float type. " + "Found " + + type + " " + name + " in file " + reader.Name()); + } + + if (type == "double" && sizeof(BoutReal) != sizeof(double)) { + throw std::invalid_argument( + "ADIOS does not allow for implicit type conversions. BoutReal type is " + "float but found " + + type + " " + name + " in file " + reader.Name()); + } + + if (type == "float" && sizeof(BoutReal) != sizeof(float)) { + throw std::invalid_argument( + "ADIOS reader does not allow for implicit type conversions. BoutReal type is " + "double but found " + + type + " " + name + " in file " + reader.Name()); + } + + auto dims = variable.Shape(); + auto ndims = dims.size(); + adios2::Variable variableD = io.InquireVariable(name); + + switch (ndims) { + case 1: { + Array value(static_cast(dims[0])); + BoutReal* data = value.begin(); + reader.Get(variableD, data, adios2::Mode::Sync); + return Options(value); + } + case 2: { + Matrix value(static_cast(dims[0]), static_cast(dims[1])); + BoutReal* data = value.begin(); + reader.Get(variableD, data, adios2::Mode::Sync); + return Options(value); + } + case 3: { + Tensor value(static_cast(dims[0]), static_cast(dims[1]), + static_cast(dims[2])); + BoutReal* data = value.begin(); + reader.Get(variableD, data, adios2::Mode::Sync); + return Options(value); + } + } + throw BoutException("ADIOS reader failed to read '{}' of dimension {} in file '{}'", + name, ndims, reader.Name()); +} + +Options readVariable(adios2::Engine& reader, adios2::IO& io, const std::string& name, + const std::string& type) { +#define declare_template_instantiation(T) \ + if (type == adios2::GetType()) { \ + return readVariable(reader, io, name, type); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_template_instantiation) + declare_template_instantiation(std::string) +#undef declare_template_instantiation + output_warn.write("ADIOS readVariable can't read type '{}' (variable '{}')", type, + name); + return Options{}; +} + +bool readAttribute(adios2::IO& io, const std::string& name, const std::string& type, + Options& result) { + // Attribute is the part of 'name' after the last '/' separator + std::string attrname; + auto pos = name.find_last_of('/'); + if (pos == std::string::npos) { + attrname = name; + } else { + attrname = name.substr(pos + 1); + } + +#define declare_template_instantiation(T) \ + if (type == adios2::GetType()) { \ + adios2::Attribute a = io.InquireAttribute(name); \ + result.attributes[attrname] = *a.Data().data(); \ + return true; \ + } + // Only some types of attributes are supported + //declare_template_instantiation(bool) + declare_template_instantiation(int) declare_template_instantiation(BoutReal) + declare_template_instantiation(std::string) +#undef declare_template_instantiation + output_warn.write("ADIOS readAttribute can't read type '{}' (variable '{}')", + type, name); + return false; +} + +Options OptionsADIOS::read() { + Timer timer("io"); + + // Open file + ADIOSPtr adiosp = GetADIOSPtr(); + adios2::IO io; + std::string ioname = "read_" + filename; + try { + io = adiosp->AtIO(ioname); + } catch (const std::invalid_argument& e) { + io = adiosp->DeclareIO(ioname); + } + + adios2::Engine reader = io.Open(filename, adios2::Mode::ReadRandomAccess); + if (!reader) { + throw BoutException("Could not open ADIOS file '{:s}' for reading", filename); + } + + Options result; + + // Iterate over all variables + for (const auto& varpair : io.AvailableVariables()) { + const auto& var_name = varpair.first; // Name of the variable + + auto it = varpair.second.find("Type"); + const std::string& var_type = it->second; + + Options* varptr = &result; + for (const auto& piece : strsplit(var_name, '/')) { + varptr = &(*varptr)[piece]; // Navigate to subsection if needed + } + Options& var = *varptr; + + // Note: Copying the value rather than simple assignment is used + // because the Options assignment operator overwrites full_name. + var = 0; // Setting is_section to false + var.value = readVariable(reader, io, var_name, var_type).value; + var.attributes["source"] = filename; + + // Get variable attributes + for (const auto& attpair : io.AvailableAttributes(var_name, "/", true)) { + const auto& att_name = attpair.first; // Attribute name + const auto& att = attpair.second; // attribute params + + auto it = att.find("Type"); + const std::string& att_type = it->second; + readAttribute(io, att_name, att_type, var); + } + } + + reader.Close(); + + return result; +} + +void OptionsADIOS::verifyTimesteps() const { + ADIOSStream& stream = ADIOSStream::ADIOSGetStream(filename); + stream.engine.EndStep(); + stream.isInStep = false; + return; +} + +/// Visit a variant type, and put the data into a NcVar +struct ADIOSPutVarVisitor { + ADIOSPutVarVisitor(const std::string& name, ADIOSStream& stream) + : varname(name), stream(stream) {} + template + void operator()(const T& value) { + adios2::Variable var = stream.GetValueVariable(varname); + stream.engine.Put(var, value); + } + +private: + const std::string& varname; + ADIOSStream& stream; +}; + +template <> +void ADIOSPutVarVisitor::operator()(const bool& value) { + // Scalars are only written from processor 0 + if (BoutComm::rank() != 0) { + return; + } + adios2::Variable var = stream.GetValueVariable(varname); + stream.engine.Put(var, static_cast(value)); +} + +template <> +void ADIOSPutVarVisitor::operator()(const int& value) { + // Scalars are only written from processor 0 + if (BoutComm::rank() != 0) { + return; + } + adios2::Variable var = stream.GetValueVariable(varname); + stream.engine.Put(var, value); +} + +template <> +void ADIOSPutVarVisitor::operator()(const BoutReal& value) { + // Scalars are only written from processor 0 + if (BoutComm::rank() != 0) { + return; + } + adios2::Variable var = stream.GetValueVariable(varname); + stream.engine.Put(var, value); +} + +template <> +void ADIOSPutVarVisitor::operator()(const std::string& value) { + // Scalars are only written from processor 0 + if (BoutComm::rank() != 0) { + return; + } + adios2::Variable var = stream.GetValueVariable(varname); + stream.engine.Put(var, value, adios2::Mode::Sync); +} + +template <> +void ADIOSPutVarVisitor::operator()(const Field2D& value) { + // Get the mesh that describes how local data relates to global arrays + auto mesh = value.getMesh(); + + // The global size of this array includes boundary cells but not communication guard cells. + // In general this array will be sparse because it may have gaps. + adios2::Dims shape = {static_cast(mesh->GlobalNx), + static_cast(mesh->GlobalNy)}; + + // Offset of this processor's data into the global array + adios2::Dims start = {static_cast(mesh->MapGlobalX), + static_cast(mesh->MapGlobalY)}; + + // The size of the mapped region + adios2::Dims count = {static_cast(mesh->MapCountX), + static_cast(mesh->MapCountY)}; + + // Where the actual data starts in data pointer (to exclude ghost cells) + adios2::Dims memStart = {static_cast(mesh->MapLocalX), + static_cast(mesh->MapLocalY)}; + + // The actual size of data pointer in memory (including ghost cells) + adios2::Dims memCount = {static_cast(value.getNx()), + static_cast(value.getNy())}; + + adios2::Variable var = stream.GetArrayVariable(varname, shape); + /* std::cout << "PutVar Field2D rank " << BoutComm::rank() << " var = " << varname + << " shape = " << shape[0] << "x" << shape[1] << " count = " << count[0] + << "x" << count[1] << " Nx*Ny = " << value.getNx() << "x" << value.getNy() + << " memStart = " << memStart[0] << "x" << memStart[1] + << " memCount = " << memCount[0] << "x" << memCount[1] << std::endl;*/ + var.SetSelection({start, count}); + var.SetMemorySelection({memStart, memCount}); + stream.engine.Put(var, &value(0, 0)); +} + +template <> +void ADIOSPutVarVisitor::operator()(const Field3D& value) { + // Get the mesh that describes how local data relates to global arrays + auto mesh = value.getMesh(); + + // The global size of this array includes boundary cells but not communication guard cells. + // In general this array will be sparse because it may have gaps. + adios2::Dims shape = {static_cast(mesh->GlobalNx), + static_cast(mesh->GlobalNy), + static_cast(mesh->GlobalNz)}; + + // Offset of this processor's data into the global array + adios2::Dims start = {static_cast(mesh->MapGlobalX), + static_cast(mesh->MapGlobalY), + static_cast(mesh->MapGlobalZ)}; + + // The size of the mapped region + adios2::Dims count = {static_cast(mesh->MapCountX), + static_cast(mesh->MapCountY), + static_cast(mesh->MapCountZ)}; + + // Where the actual data starts in data pointer (to exclude ghost cells) + adios2::Dims memStart = {static_cast(mesh->MapLocalX), + static_cast(mesh->MapLocalY), + static_cast(mesh->MapLocalZ)}; + + // The actual size of data pointer in memory (including ghost cells) + adios2::Dims memCount = {static_cast(value.getNx()), + static_cast(value.getNy()), + static_cast(value.getNz())}; + + adios2::Variable var = stream.GetArrayVariable(varname, shape); + /*std::cout << "PutVar Field3D rank " << BoutComm::rank() << " var = " << varname + << " shape = " << shape[0] << "x" << shape[1] << "x" << shape[2] + << " count = " << count[0] << "x" << count[1] << "x" << count[2] + << " Nx*Ny = " << value.getNx() << "x" << value.getNy() << "x" + << value.getNz() << " memStart = " << memStart[0] << "x" << memStart[1] << "x" + << memStart[2] << " memCount = " << memCount[0] << "x" << memCount[1] << "x" + << memCount[2] << std::endl;*/ + var.SetSelection({start, count}); + var.SetMemorySelection({memStart, memCount}); + stream.engine.Put(var, &value(0, 0, 0)); +} + +template <> +void ADIOSPutVarVisitor::operator()(const FieldPerp& value) { + // Get the mesh that describes how local data relates to global arrays + auto mesh = value.getMesh(); + + // The global size of this array includes boundary cells but not communication guard cells. + // In general this array will be sparse because it may have gaps. + adios2::Dims shape = {static_cast(mesh->GlobalNx), + static_cast(mesh->GlobalNz)}; + + // Offset of this processor's data into the global array + adios2::Dims start = {static_cast(mesh->MapGlobalX), + static_cast(mesh->MapGlobalZ)}; + + // The size of the mapped region + adios2::Dims count = {static_cast(mesh->MapCountX), + static_cast(mesh->MapCountZ)}; + + // Where the actual data starts in data pointer (to exclude ghost cells) + adios2::Dims memStart = {static_cast(mesh->MapLocalX), + static_cast(mesh->MapLocalZ)}; + + // The actual size of data pointer in memory (including ghost cells) + adios2::Dims memCount = {static_cast(value.getNx()), + static_cast(value.getNz())}; + + adios2::Variable var = stream.GetArrayVariable(varname, shape); + /* std::cout << "PutVar FieldPerp rank " << BoutComm::rank() << " var = " << varname + << " shape = " << shape[0] << "x" << shape[1] << " count = " << count[0] + << "x" << count[1] << " Nx*Ny = " << value.getNx() << "x" << value.getNy() + << " memStart = " << memStart[0] << "x" << memStart[1] + << " memCount = " << memCount[0] << "x" << memCount[1] << std::endl; */ + var.SetSelection({start, count}); + var.SetMemorySelection({memStart, memCount}); + stream.engine.Put(var, &value(0, 0)); +} + +template <> +void ADIOSPutVarVisitor::operator()>(const Array& value) { + // Pointer to data. Assumed to be contiguous array + adios2::Dims shape = {(size_t)BoutComm::size(), (size_t)value.size()}; + adios2::Dims start = {(size_t)BoutComm::rank(), 0}; + adios2::Dims count = {1, shape[1]}; + adios2::Variable var = stream.GetArrayVariable(varname, shape); + var.SetSelection({start, count}); + stream.engine.Put(var, value.begin()); +} + +template <> +void ADIOSPutVarVisitor::operator()>(const Matrix& value) { + // Pointer to data. Assumed to be contiguous array + auto s = value.shape(); + adios2::Dims shape = {(size_t)BoutComm::size(), (size_t)std::get<0>(s), + (size_t)std::get<1>(s)}; + adios2::Dims start = {(size_t)BoutComm::rank(), 0, 0}; + adios2::Dims count = {1, shape[1], shape[2]}; + adios2::Variable var = stream.GetArrayVariable(varname, shape); + var.SetSelection({start, count}); + stream.engine.Put(var, value.begin()); +} + +template <> +void ADIOSPutVarVisitor::operator()>(const Tensor& value) { + // Pointer to data. Assumed to be contiguous array + auto s = value.shape(); + adios2::Dims shape = {(size_t)BoutComm::size(), (size_t)std::get<0>(s), + (size_t)std::get<1>(s), (size_t)std::get<2>(s)}; + adios2::Dims start = {(size_t)BoutComm::rank(), 0, 0, 0}; + adios2::Dims count = {1, shape[1], shape[2], shape[3]}; + adios2::Variable var = stream.GetArrayVariable(varname, shape); + var.SetSelection({start, count}); + stream.engine.Put(var, value.begin()); +} + +/// Visit a variant type, and put the data into a NcVar +struct ADIOSPutAttVisitor { + ADIOSPutAttVisitor(const std::string& varname, const std::string& attrname, + ADIOSStream& stream) + : varname(varname), attrname(attrname), stream(stream) {} + template + void operator()(const T& value) { + stream.io.DefineAttribute(attrname, value, varname, "/", false); + } + +private: + const std::string& varname; + const std::string& attrname; + ADIOSStream& stream; +}; + +template <> +void ADIOSPutAttVisitor::operator()(const bool& value) { + stream.io.DefineAttribute(attrname, (int)value, varname, "/", false); +} + +void writeGroup(const Options& options, ADIOSStream& stream, const std::string& groupname, + const std::string& time_dimension) { + + for (const auto& childpair : options.getChildren()) { + const auto& name = childpair.first; + const auto& child = childpair.second; + + if (child.isSection()) { + TRACE("Writing group '{:s}'", name); + writeGroup(child, stream, name, time_dimension); + continue; + } + + if (child.isValue()) { + try { + auto time_it = child.attributes.find("time_dimension"); + if (time_it == child.attributes.end()) { + if (stream.adiosStep > 0) { + // we should only write the non-varying values in the first step + continue; + } + } else { + // Has a time dimension + + const auto& time_name = bout::utils::get(time_it->second); + + // Only write time-varying values that match current time + // dimension being written + if (time_name != time_dimension) { + continue; + } + } + + // Write the variable + // Note: ADIOS2 uses '/' to as a group separator; BOUT++ uses ':' + std::string varname = groupname.empty() ? name : groupname + "/" + name; + bout::utils::visit(ADIOSPutVarVisitor(varname, stream), child.value); + + // Write attributes + if (!BoutComm::rank()) { + for (const auto& attribute : child.attributes) { + const std::string& att_name = attribute.first; + const auto& att = attribute.second; + + bout::utils::visit(ADIOSPutAttVisitor(varname, att_name, stream), att); + } + } + + } catch (const std::exception& e) { + throw BoutException("Error while writing value '{:s}' : {:s}", name, e.what()); + } + } + } +} + +/// Write options to file +void OptionsADIOS::write(const Options& options, const std::string& time_dim) { + Timer timer("io"); + + // ADIOSStream is just a BOUT++ object, it does not create anything inside ADIOS + ADIOSStream& stream = ADIOSStream::ADIOSGetStream(filename); + + // Need to have an adios2::IO object first, which can only be created once. + if (!stream.io) { + ADIOSPtr adiosp = GetADIOSPtr(); + std::string ioname = "write_" + filename; + try { + stream.io = adiosp->AtIO(ioname); + } catch (const std::invalid_argument& e) { + stream.io = adiosp->DeclareIO(ioname); + stream.io.SetEngine("BP5"); + } + } + + /* Open file once and keep it open, close in stream desctructor + or close after writing if singleWriteFile == true + */ + if (!stream.engine) { + adios2::Mode amode = + (file_mode == FileMode::append ? adios2::Mode::Append : adios2::Mode::Write); + stream.engine = stream.io.Open(filename, amode); + if (!stream.engine) { + throw BoutException("Could not open ADIOS file '{:s}' for writing", filename); + } + } + + /* Multiple write() calls allowed in a single adios step to output multiple + Options objects in the same step. verifyTimesteps() will indicate the + completion of the step (and adios will publish the step). + */ + if (!stream.isInStep) { + stream.engine.BeginStep(); + stream.isInStep = true; + stream.adiosStep = stream.engine.CurrentStep(); + } + + writeGroup(options, stream, "", time_dim); + + /* In singleWriteFile mode, we complete the step and close the file */ + if (singleWriteFile) { + stream.engine.EndStep(); + stream.engine.Close(); + } +} + +} // namespace bout + +#endif // BOUT_HAS_ADIOS diff --git a/src/sys/options/options_adios.hxx b/src/sys/options/options_adios.hxx new file mode 100644 index 0000000000..eddb3976ff --- /dev/null +++ b/src/sys/options/options_adios.hxx @@ -0,0 +1,83 @@ + +#pragma once + +#ifndef OPTIONS_ADIOS_H +#define OPTIONS_ADIOS_H + +#include "bout/build_config.hxx" +#include "bout/options.hxx" +#include "bout/options_io.hxx" + +#if !BOUT_HAS_ADIOS + +namespace { +bout::RegisterUnavailableOptionsIO + registerunavailableoptionsadios("adios", "BOUT++ was not configured with ADIOS2"); +} + +#else + +#include +#include + +namespace bout { + +/// Forward declare ADIOS file type so we don't need to depend +/// directly on ADIOS +struct ADIOSStream; + +class OptionsADIOS : public OptionsIO { +public: + // Constructors need to be defined in implementation due to forward + // declaration of ADIOSStream + OptionsADIOS() = delete; + + /// Create an OptionsADIOS + /// + /// Options: + /// - "file" The name of the file + /// If not set then "path" and "prefix" must be set, + /// and file is set to {path}/{prefix}.bp + /// - "append" + /// - "singleWriteFile" + OptionsADIOS(Options& options); + + OptionsADIOS(const OptionsADIOS&) = delete; + OptionsADIOS(OptionsADIOS&&) noexcept = default; + ~OptionsADIOS() = default; + + OptionsADIOS& operator=(const OptionsADIOS&) = delete; + OptionsADIOS& operator=(OptionsADIOS&&) noexcept = default; + + /// Read options from file + Options read() override; + + /// Write options to file + void write(const Options& options, const std::string& time_dim) override; + + /// Check that all variables with the same time dimension have the + /// same size in that dimension. Throws BoutException if there are + /// any differences, otherwise is silent + void verifyTimesteps() const override; + +private: + enum class FileMode { + replace, ///< Overwrite file when writing + append ///< Append to file when writing + }; + + /// Name of the file on disk + std::string filename; + /// How to open the file for writing + FileMode file_mode{FileMode::replace}; + bool singleWriteFile = false; +}; + +namespace { +RegisterOptionsIO registeroptionsadios("adios"); +} + +} // namespace bout + +#endif // BOUT_HAS_ADIOS +#endif // OPTIONS_ADIOS_H diff --git a/src/sys/options/options_io.cxx b/src/sys/options/options_io.cxx new file mode 100644 index 0000000000..6717b6b07d --- /dev/null +++ b/src/sys/options/options_io.cxx @@ -0,0 +1,58 @@ +#include "bout/options_io.hxx" +#include "bout/bout.hxx" +#include "bout/globals.hxx" +#include "bout/mesh.hxx" + +#include "options_adios.hxx" +#include "options_netcdf.hxx" + +namespace bout { +std::unique_ptr OptionsIO::create(const std::string& file) { + return OptionsIOFactory::getInstance().createFile(file); +} + +std::unique_ptr OptionsIO::create(Options& config) { + auto& factory = OptionsIOFactory::getInstance(); + return factory.create(factory.getType(&config), config); +} + +OptionsIOFactory::ReturnType OptionsIOFactory::createRestart(Options* optionsptr) const { + Options& options = optionsptr ? *optionsptr : Options::root()["restart_files"]; + + // Set defaults + options["path"].overrideDefault( + Options::root()["datadir"].withDefault("data")); + options["prefix"].overrideDefault("BOUT.restart"); + options["append"].overrideDefault(false); + options["singleWriteFile"].overrideDefault(true); + return create(getType(&options), options); +} + +OptionsIOFactory::ReturnType OptionsIOFactory::createOutput(Options* optionsptr) const { + Options& options = optionsptr ? *optionsptr : Options::root()["output"]; + + // Set defaults + options["path"].overrideDefault( + Options::root()["datadir"].withDefault("data")); + options["prefix"].overrideDefault("BOUT.dmp"); + options["append"].overrideDefault(Options::root()["append"] + .doc("Add output data to existing (dump) files?") + .withDefault(false)); + return create(getType(&options), options); +} + +OptionsIOFactory::ReturnType OptionsIOFactory::createFile(const std::string& file) const { + Options options{{"file", file}}; + return create(getDefaultType(), options); +} + +void writeDefaultOutputFile(Options& data) { + // Add BOUT++ version and flags + bout::experimental::addBuildFlagsToOptions(data); + // Add mesh information + bout::globals::mesh->outputVars(data); + // Write to the default output file + OptionsIOFactory::getInstance().createOutput()->write(data); +} + +} // namespace bout diff --git a/src/sys/options/options_netcdf.cxx b/src/sys/options/options_netcdf.cxx index 7c21b309bc..fb6efd6567 100644 --- a/src/sys/options/options_netcdf.cxx +++ b/src/sys/options/options_netcdf.cxx @@ -2,7 +2,7 @@ #if BOUT_HAS_NETCDF && !BOUT_HAS_LEGACY_NETCDF -#include "bout/options_netcdf.hxx" +#include "options_netcdf.hxx" #include "bout/bout.hxx" #include "bout/globals.hxx" @@ -647,14 +647,19 @@ std::vector verifyTimesteps(const NcGroup& group) { namespace bout { -OptionsNetCDF::OptionsNetCDF() : data_file(nullptr) {} - -OptionsNetCDF::OptionsNetCDF(std::string filename, FileMode mode) - : filename(std::move(filename)), file_mode(mode), data_file(nullptr) {} +OptionsNetCDF::OptionsNetCDF(Options& options) : OptionsIO(options) { + if (options["file"].doc("File name. Defaults to /..nc").isSet()) { + filename = options["file"].as(); + } else { + // Both path and prefix must be set + filename = fmt::format("{}/{}.{}.nc", options["path"].as(), + options["prefix"].as(), BoutComm::rank()); + } -OptionsNetCDF::~OptionsNetCDF() = default; -OptionsNetCDF::OptionsNetCDF(OptionsNetCDF&&) noexcept = default; -OptionsNetCDF& OptionsNetCDF::operator=(OptionsNetCDF&&) noexcept = default; + file_mode = (options["append"].doc("Append to existing file?").withDefault(false)) + ? FileMode::append + : FileMode::replace; +} void OptionsNetCDF::verifyTimesteps() const { NcFile dataFile(filename, NcFile::read); @@ -703,41 +708,6 @@ void OptionsNetCDF::write(const Options& options, const std::string& time_dim) { data_file->sync(); } -std::string getRestartDirectoryName(Options& options) { - if (options["restartdir"].isSet()) { - // Solver-specific restart directory - return options["restartdir"].withDefault("data"); - } - // Use the root data directory - return options["datadir"].withDefault("data"); -} - -std::string getRestartFilename(Options& options) { - return getRestartFilename(options, BoutComm::rank()); -} - -std::string getRestartFilename(Options& options, int rank) { - return fmt::format("{}/BOUT.restart.{}.nc", bout::getRestartDirectoryName(options), - rank); -} - -std::string getOutputFilename(Options& options) { - return getOutputFilename(options, BoutComm::rank()); -} - -std::string getOutputFilename(Options& options, int rank) { - return fmt::format("{}/BOUT.dmp.{}.nc", - options["datadir"].withDefault("data"), rank); -} - -void writeDefaultOutputFile() { writeDefaultOutputFile(Options::root()); } - -void writeDefaultOutputFile(Options& options) { - bout::experimental::addBuildFlagsToOptions(options); - bout::globals::mesh->outputVars(options); - OptionsNetCDF(getOutputFilename(Options::root())).write(options); -} - } // namespace bout #endif // BOUT_HAS_NETCDF diff --git a/src/sys/options/options_netcdf.hxx b/src/sys/options/options_netcdf.hxx new file mode 100644 index 0000000000..8f195c9d92 --- /dev/null +++ b/src/sys/options/options_netcdf.hxx @@ -0,0 +1,84 @@ + +#pragma once + +#ifndef OPTIONS_NETCDF_H +#define OPTIONS_NETCDF_H + +#include "bout/build_config.hxx" + +#include "bout/options.hxx" +#include "bout/options_io.hxx" + +#if !BOUT_HAS_NETCDF || BOUT_HAS_LEGACY_NETCDF + +namespace { +RegisterUnavailableOptionsIO + registerunavailableoptionsnetcdf("netcdf", "BOUT++ was not configured with NetCDF"); +} + +#else + +#include +#include +#include + +namespace bout { + +class OptionsNetCDF : public OptionsIO { +public: + // Constructors need to be defined in implementation due to forward + // declaration of NcFile + OptionsNetCDF() = delete; + + /// Create an OptionsNetCDF + /// + /// Options: + /// - "file" The name of the file + /// If not set then "path" and "prefix" options must be set, + /// and file is set to {path}/{prefix}.{rank}.nc + /// - "append" File mode, default is false + OptionsNetCDF(Options& options); + + ~OptionsNetCDF() {} + + OptionsNetCDF(const OptionsNetCDF&) = delete; + OptionsNetCDF(OptionsNetCDF&&) noexcept = default; + OptionsNetCDF& operator=(const OptionsNetCDF&) = delete; + OptionsNetCDF& operator=(OptionsNetCDF&&) noexcept = default; + + /// Read options from file + Options read(); + + /// Write options to file + void write(const Options& options) { write(options, "t"); } + void write(const Options& options, const std::string& time_dim); + + /// Check that all variables with the same time dimension have the + /// same size in that dimension. Throws BoutException if there are + /// any differences, otherwise is silent + void verifyTimesteps() const; + +private: + enum class FileMode { + replace, ///< Overwrite file when writing + append ///< Append to file when writing + }; + + /// Pointer to netCDF file so we don't introduce direct dependence + std::unique_ptr data_file = nullptr; + + /// Name of the file on disk + std::string filename; + /// How to open the file for writing + FileMode file_mode{FileMode::replace}; +}; + +namespace { +RegisterOptionsIO registeroptionsnetcdf("netcdf"); +} + +} // namespace bout + +#endif + +#endif // OPTIONS_NETCDF_H diff --git a/tests/MMS/spatial/fci/data/BOUT.inp b/tests/MMS/spatial/fci/data/BOUT.inp index 5f377bbb56..802f70f99b 100644 --- a/tests/MMS/spatial/fci/data/BOUT.inp +++ b/tests/MMS/spatial/fci/data/BOUT.inp @@ -5,6 +5,7 @@ input_field = sin(y - 2*z) + sin(y - z) solution = (6.28318530717959*(0.01*x + 0.045)*(-2*cos(y - 2*z) - cos(y - z)) + 0.628318530717959*cos(y - 2*z) + 0.628318530717959*cos(y - z))/sqrt((0.01*x + 0.045)^2 + 1.0) MXG = 1 +MYG = 1 NXPE = 1 [mesh] diff --git a/tests/MMS/spatial/fci/fci_mms.cxx b/tests/MMS/spatial/fci/fci_mms.cxx index 5a2599368e..18405a7f88 100644 --- a/tests/MMS/spatial/fci/fci_mms.cxx +++ b/tests/MMS/spatial/fci/fci_mms.cxx @@ -17,6 +17,8 @@ int main(int argc, char** argv) { Field3D error{result - solution}; Options dump; + // Add mesh geometry variables + mesh->outputVars(dump); dump["l_2"] = sqrt(mean(SQ(error), true, "RGN_NOBNDRY")); dump["l_inf"] = max(abs(error), true, "RGN_NOBNDRY"); diff --git a/tests/integrated/CMakeLists.txt b/tests/integrated/CMakeLists.txt index 8dd892ebad..7d3e8e81ce 100644 --- a/tests/integrated/CMakeLists.txt +++ b/tests/integrated/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory(test-laplacexz) add_subdirectory(test-multigrid_laplace) add_subdirectory(test-naulin-laplace) add_subdirectory(test-options-netcdf) +add_subdirectory(test-options-adios) add_subdirectory(test-petsc_laplace) add_subdirectory(test-petsc_laplace_MAST-grid) add_subdirectory(test-restart-io) diff --git a/tests/integrated/test-beuler/test_beuler.cxx b/tests/integrated/test-beuler/test_beuler.cxx index cfdae89eb2..5a080767dd 100644 --- a/tests/integrated/test-beuler/test_beuler.cxx +++ b/tests/integrated/test-beuler/test_beuler.cxx @@ -41,7 +41,6 @@ class TestSolver : public PhysicsModel { }; int main(int argc, char** argv) { - // Absolute tolerance for difference between the actual value and the // expected value constexpr BoutReal tolerance = 1.e-5; @@ -87,8 +86,6 @@ int main(int argc, char** argv) { solver->solve(); - BoutFinalise(false); - if (model.check_solution(tolerance)) { output_test << " PASSED\n"; return 0; diff --git a/tests/integrated/test-griddata/test_griddata.cxx b/tests/integrated/test-griddata/test_griddata.cxx index 9f12d48d9d..0277f9e001 100644 --- a/tests/integrated/test-griddata/test_griddata.cxx +++ b/tests/integrated/test-griddata/test_griddata.cxx @@ -13,7 +13,7 @@ int main(int argc, char** argv) { dump["Bpxy"] = Bpxy; bout::experimental::addBuildFlagsToOptions(dump); bout::globals::mesh->outputVars(dump); - bout::OptionsNetCDF("data.nc").write(dump); + bout::OptionsIO::create("data.nc")->write(dump); BoutFinalise(); return 0; diff --git a/tests/integrated/test-options-adios/CMakeLists.txt b/tests/integrated/test-options-adios/CMakeLists.txt new file mode 100644 index 0000000000..110773d6fd --- /dev/null +++ b/tests/integrated/test-options-adios/CMakeLists.txt @@ -0,0 +1,6 @@ +bout_add_integrated_test(test-options-adios + SOURCES test-options-adios.cxx + USE_RUNTEST + USE_DATA_BOUT_INP + REQUIRES BOUT_HAS_ADIOS + ) diff --git a/tests/integrated/test-options-adios/data/BOUT.inp b/tests/integrated/test-options-adios/data/BOUT.inp new file mode 100644 index 0000000000..fa0f6d3681 --- /dev/null +++ b/tests/integrated/test-options-adios/data/BOUT.inp @@ -0,0 +1,6 @@ + + +[mesh] +nx = 5 +ny = 2 +nz = 2 diff --git a/tests/integrated/test-options-adios/makefile b/tests/integrated/test-options-adios/makefile new file mode 100644 index 0000000000..7dbbce8736 --- /dev/null +++ b/tests/integrated/test-options-adios/makefile @@ -0,0 +1,6 @@ + +BOUT_TOP = ../../.. + +SOURCEC = test-options-adios.cxx + +include $(BOUT_TOP)/make.config diff --git a/tests/integrated/test-options-adios/runtest b/tests/integrated/test-options-adios/runtest new file mode 100755 index 0000000000..1621c686a3 --- /dev/null +++ b/tests/integrated/test-options-adios/runtest @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +# Note: This test requires NCDF4, whereas on Travis NCDF is used +# requires: netcdf +# requires: adios +# requires: not legacy_netcdf + +from boututils.datafile import DataFile +from boututils.run_wrapper import build_and_log, shell, launch +from boutdata.data import BoutOptionsFile + +import math +import numpy as np + +build_and_log("options-netcdf test") +shell("rm -f test-out.ini") +shell("rm -f test-out.nc") + +# Create a NetCDF input file +with DataFile("test.nc", create=True, format="NETCDF4") as f: + f.write("int", 42) + f.write("real", 3.1415) + f.write("string", "hello") + +# run BOUT++ +launch("./test-options-adios", nproc=1, mthread=1) + +# Check the output INI file +result = BoutOptionsFile("test-out.ini") + +print(result) + +assert result["int"] == 42 +assert math.isclose(result["real"], 3.1415) +assert result["string"] == "hello" + +print("Checking saved ADIOS test-out file -- Not implemented") + +# Check the output NetCDF file +# with DataFile("test-out.nc") as f: +# assert f["int"] == 42 +# assert math.isclose(f["real"], 3.1415) +# assert result["string"] == "hello" + +print("Checking saved settings.ini") + +# Check the settings.ini file, coming from BOUT.inp +# which is converted to NetCDF, read in, then written again +settings = BoutOptionsFile("settings.ini") + +assert settings["mesh"]["nx"] == 5 +assert settings["mesh"]["ny"] == 2 + +print("Checking saved fields.bp -- Not implemented") + +# with DataFile("fields.nc") as f: +# assert f["f2d"].shape == (5, 6) # Field2D +# assert f["f3d"].shape == (5, 6, 2) # Field3D +# assert f["fperp"].shape == (5, 2) # FieldPerp +# assert np.allclose(f["f2d"], 1.0) +# assert np.allclose(f["f3d"], 2.0) +# assert np.allclose(f["fperp"], 3.0) + +print("Checking saved fields2.bp -- Not implemented") + +# with DataFile("fields2.nc") as f: +# assert f["f2d"].shape == (5, 6) # Field2D +# assert f["f3d"].shape == (5, 6, 2) # Field3D +# assert f["fperp"].shape == (5, 2) # FieldPerp +# assert np.allclose(f["f2d"], 1.0) +# assert np.allclose(f["f3d"], 2.0) +# assert np.allclose(f["fperp"], 3.0) + +print(" => Passed") diff --git a/tests/integrated/test-options-adios/test-options-adios.cxx b/tests/integrated/test-options-adios/test-options-adios.cxx new file mode 100644 index 0000000000..60604e1aa3 --- /dev/null +++ b/tests/integrated/test-options-adios/test-options-adios.cxx @@ -0,0 +1,111 @@ + +#include "bout/bout.hxx" + +#include "bout/options_io.hxx" +#include "bout/optionsreader.hxx" + +using bout::OptionsIO; + +int main(int argc, char** argv) { + BoutInitialise(argc, argv); + + // Read values from a NetCDF file + auto file = bout::OptionsIO::create("test.nc"); + + auto values = file->read(); + + values.printUnused(); + + // Write to an INI text file + OptionsReader* reader = OptionsReader::getInstance(); + reader->write(&values, "test-out.ini"); + + // Write to ADIOS file, by setting file type "adios" + OptionsIO::create({{"file", "test-out.bp"}, + {"type", "adios"}, + {"append", false}, + {"singleWriteFile", true}}) + ->write(values); + + /////////////////////////// + + // Write the BOUT.inp settings to ADIOS file + OptionsIO::create({{"file", "settings.bp"}, + {"type", "adios"}, + {"append", false}, + {"singleWriteFile", true}}) + ->write(Options::root()); + + // Read back in + auto settings = OptionsIO::create({{"file", "settings.bp"}, {"type", "adios"}})->read(); + + // Write to INI file + reader->write(&settings, "settings.ini"); + + /////////////////////////// + // Write fields + + Options fields; + fields["f2d"] = Field2D(1.0); + fields["f3d"] = Field3D(2.0); + fields["fperp"] = FieldPerp(3.0); + auto f = OptionsIO::create({{"file", "fields.bp"}, {"type", "adios"}}); + /* + write() for adios only buffers data but does not guarantee writing to disk + unless singleWriteFile is set to true + */ + f->write(fields); + // indicate completion of step, required to get data on disk + f->verifyTimesteps(); + + /////////////////////////// + // Read fields + + Options fields_in = + OptionsIO::create({{"file", "fields.bp"}, {"type", "adios"}})->read(); + + auto f2d = fields_in["f2d"].as(bout::globals::mesh); + auto f3d = fields_in["f3d"].as(bout::globals::mesh); + auto fperp = fields_in["fperp"].as(bout::globals::mesh); + + Options fields2; + fields2["f2d"] = f2d; + fields2["f3d"] = f3d; + fields2["fperp"] = fperp; + + // Write out again + auto f2 = OptionsIO::create({{"file", "fields2.bp"}, + {"type", "adios"}, + {"append", false}, + {"singleWriteFile", true}}); + f2->write(fields2); + + /////////////////////////// + // Time dependent values + + Options data; + data["scalar"] = 1.0; + data["scalar"].attributes["time_dimension"] = "t"; + + data["field"] = Field3D(2.0); + data["field"].attributes["time_dimension"] = "t"; + + OptionsIO::create({{"file", "time.bp"}, + {"type", "adios"}, + {"append", false}, + {"singleWriteFile", true}}) + ->write(data); + + // Update time-dependent values + data["scalar"] = 2.0; + data["field"] = Field3D(3.0); + + // Append data to file + OptionsIO::create({{"file", "time.bp"}, + {"type", "adios"}, + {"append", true}, + {"singleWriteFile", true}}) + ->write(data); + + BoutFinalise(); +}; diff --git a/tests/integrated/test-options-netcdf/test-options-netcdf.cxx b/tests/integrated/test-options-netcdf/test-options-netcdf.cxx index 01c5749972..f5daf8919c 100644 --- a/tests/integrated/test-options-netcdf/test-options-netcdf.cxx +++ b/tests/integrated/test-options-netcdf/test-options-netcdf.cxx @@ -1,18 +1,18 @@ #include "bout/bout.hxx" -#include "bout/options_netcdf.hxx" +#include "bout/options_io.hxx" #include "bout/optionsreader.hxx" -using bout::OptionsNetCDF; +using bout::OptionsIO; int main(int argc, char** argv) { BoutInitialise(argc, argv); // Read values from a NetCDF file - OptionsNetCDF file("test.nc"); + auto file = OptionsIO::create("test.nc"); - auto values = file.read(); + auto values = file->read(); values.printUnused(); @@ -21,15 +21,15 @@ int main(int argc, char** argv) { reader->write(&values, "test-out.ini"); // Write to a NetCDF file - OptionsNetCDF("test-out.nc").write(values); + OptionsIO::create("test-out.nc")->write(values); /////////////////////////// // Write the BOUT.inp settings to NetCDF file - OptionsNetCDF("settings.nc").write(Options::root()); + OptionsIO::create("settings.nc")->write(Options::root()); // Read back in - auto settings = OptionsNetCDF("settings.nc").read(); + auto settings = OptionsIO::create("settings.nc")->read(); // Write to INI file reader->write(&settings, "settings.ini"); @@ -41,12 +41,12 @@ int main(int argc, char** argv) { fields["f2d"] = Field2D(1.0); fields["f3d"] = Field3D(2.0); fields["fperp"] = FieldPerp(3.0); - OptionsNetCDF("fields.nc").write(fields); + OptionsIO::create("fields.nc")->write(fields); /////////////////////////// // Read fields - Options fields_in = OptionsNetCDF("fields.nc").read(); + Options fields_in = OptionsIO::create("fields.nc")->read(); auto f2d = fields_in["f2d"].as(bout::globals::mesh); auto f3d = fields_in["f3d"].as(bout::globals::mesh); @@ -58,7 +58,7 @@ int main(int argc, char** argv) { fields2["fperp"] = fperp; // Write out again - OptionsNetCDF("fields2.nc").write(fields2); + OptionsIO::create("fields2.nc")->write(fields2); /////////////////////////// // Time dependent values @@ -70,14 +70,14 @@ int main(int argc, char** argv) { data["field"] = Field3D(2.0); data["field"].attributes["time_dimension"] = "t"; - OptionsNetCDF("time.nc").write(data); + OptionsIO::create("time.nc")->write(data); // Update time-dependent values data["scalar"] = 2.0; data["field"] = Field3D(3.0); // Append data to file - OptionsNetCDF("time.nc", OptionsNetCDF::FileMode::append).write(data); + OptionsIO::create({{"file", "time.nc"}, {"append", true}})->write(data); BoutFinalise(); }; diff --git a/tests/integrated/test-solver/test_solver.cxx b/tests/integrated/test-solver/test_solver.cxx index ee64c6097f..2e4345c8cc 100644 --- a/tests/integrated/test-solver/test_solver.cxx +++ b/tests/integrated/test-solver/test_solver.cxx @@ -146,8 +146,6 @@ int main(int argc, char** argv) { } } - BoutFinalise(false); - if (!errors.empty()) { output_test << "\n => Some failed tests\n\n"; for (auto& error : errors) { diff --git a/tests/integrated/test-twistshift-staggered/test-twistshift.cxx b/tests/integrated/test-twistshift-staggered/test-twistshift.cxx index 87b9e0a094..c9f8be7b09 100644 --- a/tests/integrated/test-twistshift-staggered/test-twistshift.cxx +++ b/tests/integrated/test-twistshift-staggered/test-twistshift.cxx @@ -30,7 +30,7 @@ int main(int argc, char** argv) { Options::root()["test"] = test; Options::root()["test_aligned"] = test_aligned; - bout::writeDefaultOutputFile(); + bout::writeDefaultOutputFile(Options::root()); BoutFinalise(); } diff --git a/tests/integrated/test-twistshift/test-twistshift.cxx b/tests/integrated/test-twistshift/test-twistshift.cxx index ccde8b82b6..2c0fc79563 100644 --- a/tests/integrated/test-twistshift/test-twistshift.cxx +++ b/tests/integrated/test-twistshift/test-twistshift.cxx @@ -28,7 +28,7 @@ int main(int argc, char** argv) { Options::root()["test"] = test; Options::root()["test_aligned"] = test_aligned; - bout::writeDefaultOutputFile(); + bout::writeDefaultOutputFile(Options::root()); BoutFinalise(); } diff --git a/tests/integrated/test-yupdown-weights/test_yupdown_weights.cxx b/tests/integrated/test-yupdown-weights/test_yupdown_weights.cxx index 22d1fb9e07..e8a2982bfd 100644 --- a/tests/integrated/test-yupdown-weights/test_yupdown_weights.cxx +++ b/tests/integrated/test-yupdown-weights/test_yupdown_weights.cxx @@ -70,7 +70,7 @@ int main(int argc, char** argv) { Options::root()["ddy"] = ddy; Options::root()["ddy2"] = ddy2; - bout::writeDefaultOutputFile(); + bout::writeDefaultOutputFile(Options::root()); BoutFinalise(); diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c0fd4a1e1f..c4ffa4fa75 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,6 +19,9 @@ if(NOT TARGET gtest) message(FATAL_ERROR "googletest not found! Have you disabled the git submodules (GIT_SUBMODULE)?") endif() +# Some unit tests require GMOCK, so make sure we build it +set(BUILD_GMOCK ON) + mark_as_advanced( BUILD_GMOCK BUILD_GTEST BUILD_SHARED_LIBS gmock_build_tests gtest_build_samples gtest_build_tests diff --git a/tests/unit/sys/test_options_netcdf.cxx b/tests/unit/sys/test_options_netcdf.cxx index b086043822..5869cf4932 100644 --- a/tests/unit/sys/test_options_netcdf.cxx +++ b/tests/unit/sys/test_options_netcdf.cxx @@ -9,9 +9,9 @@ #include "test_extras.hxx" #include "bout/field3d.hxx" #include "bout/mesh.hxx" -#include "bout/options_netcdf.hxx" +#include "bout/options_io.hxx" -using bout::OptionsNetCDF; +using bout::OptionsIO; #include @@ -39,11 +39,11 @@ TEST_F(OptionsNetCDFTest, ReadWriteInt) { options["test"] = 42; // Write the file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read again - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["test"], 42); } @@ -54,11 +54,11 @@ TEST_F(OptionsNetCDFTest, ReadWriteString) { options["test"] = std::string{"hello"}; // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["test"], std::string("hello")); } @@ -69,11 +69,11 @@ TEST_F(OptionsNetCDFTest, ReadWriteField2D) { options["test"] = Field2D(1.0); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); Field2D value = data["test"].as(bout::globals::mesh); @@ -87,11 +87,11 @@ TEST_F(OptionsNetCDFTest, ReadWriteField3D) { options["test"] = Field3D(2.4); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); Field3D value = data["test"].as(bout::globals::mesh); @@ -106,11 +106,11 @@ TEST_F(OptionsNetCDFTest, Groups) { options["test"]["key"] = 42; // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["test"]["key"], 42); } @@ -121,11 +121,11 @@ TEST_F(OptionsNetCDFTest, AttributeInt) { options["test"].attributes["thing"] = 4; // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["test"].attributes["thing"].as(), 4); } @@ -136,11 +136,11 @@ TEST_F(OptionsNetCDFTest, AttributeBoutReal) { options["test"].attributes["thing"] = 3.14; // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_DOUBLE_EQ(data["test"].attributes["thing"].as(), 3.14); } @@ -151,11 +151,11 @@ TEST_F(OptionsNetCDFTest, AttributeString) { options["test"].attributes["thing"] = "hello"; // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["test"].attributes["thing"].as(), "hello"); } @@ -165,11 +165,11 @@ TEST_F(OptionsNetCDFTest, Field2DWriteCellCentre) { options["f2d"] = Field2D(2.0); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["f2d"].attributes["cell_location"].as(), toString(CELL_CENTRE)); @@ -181,11 +181,11 @@ TEST_F(OptionsNetCDFTest, Field2DWriteCellYLow) { options["f2d"] = Field2D(2.0, mesh_staggered).setLocation(CELL_YLOW); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["f2d"].attributes["cell_location"].as(), toString(CELL_YLOW)); @@ -197,11 +197,11 @@ TEST_F(OptionsNetCDFTest, Field3DWriteCellCentre) { options["f3d"] = Field3D(2.0); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["f3d"].attributes["cell_location"].as(), toString(CELL_CENTRE)); @@ -213,11 +213,11 @@ TEST_F(OptionsNetCDFTest, Field3DWriteCellYLow) { options["f3d"] = Field3D(2.0, mesh_staggered).setLocation(CELL_YLOW); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["f3d"].attributes["cell_location"].as(), toString(CELL_YLOW)); @@ -235,11 +235,11 @@ TEST_F(OptionsNetCDFTest, FieldPerpWriteCellCentre) { fperp.getMesh()->getXcomm(); // Write file - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } // Read file - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_EQ(data["fperp"].attributes["cell_location"].as(), toString(CELL_CENTRE)); @@ -252,10 +252,10 @@ TEST_F(OptionsNetCDFTest, VerifyTimesteps) { options["thing1"] = 1.0; options["thing1"].attributes["time_dimension"] = "t"; - OptionsNetCDF(filename).write(options); + OptionsIO::create(filename)->write(options); } - EXPECT_NO_THROW(OptionsNetCDF(filename).verifyTimesteps()); + EXPECT_NO_THROW(OptionsIO::create(filename)->verifyTimesteps()); { Options options; @@ -265,10 +265,11 @@ TEST_F(OptionsNetCDFTest, VerifyTimesteps) { options["thing2"] = 3.0; options["thing2"].attributes["time_dimension"] = "t"; - OptionsNetCDF(filename, OptionsNetCDF::FileMode::append).write(options); + OptionsIO::create({{"type", "netcdf"}, {"file", filename}, {"append", true}}) + ->write(options); } - EXPECT_THROW(OptionsNetCDF(filename).verifyTimesteps(), BoutException); + EXPECT_THROW(OptionsIO::create(filename)->verifyTimesteps(), BoutException); } TEST_F(OptionsNetCDFTest, WriteTimeDimension) { @@ -278,10 +279,10 @@ TEST_F(OptionsNetCDFTest, WriteTimeDimension) { options["thing2"].assignRepeat(2.0, "t2"); // non-default // Only write non-default time dim - OptionsNetCDF(filename).write(options, "t2"); + OptionsIO::create(filename)->write(options, "t2"); } - Options data = OptionsNetCDF(filename).read(); + Options data = OptionsIO::create(filename)->read(); EXPECT_FALSE(data.isSet("thing1")); EXPECT_TRUE(data.isSet("thing2")); @@ -297,12 +298,12 @@ TEST_F(OptionsNetCDFTest, WriteMultipleTimeDimensions) { options["thing4_t2"].assignRepeat(2.0, "t2"); // non-default // Write the non-default time dim twice - OptionsNetCDF(filename).write(options, "t2"); - OptionsNetCDF(filename).write(options, "t2"); - OptionsNetCDF(filename).write(options, "t"); + OptionsIO::create(filename)->write(options, "t2"); + OptionsIO::create(filename)->write(options, "t2"); + OptionsIO::create(filename)->write(options, "t"); } - EXPECT_NO_THROW(OptionsNetCDF(filename).verifyTimesteps()); + EXPECT_NO_THROW(OptionsIO::create(filename)->verifyTimesteps()); } #endif // BOUT_HAS_NETCDF diff --git a/tools/pylib/_boutpp_build/bout_options.pxd b/tools/pylib/_boutpp_build/bout_options.pxd index ba5e64c8e3..be17608cea 100644 --- a/tools/pylib/_boutpp_build/bout_options.pxd +++ b/tools/pylib/_boutpp_build/bout_options.pxd @@ -8,20 +8,11 @@ cdef extern from "boutexception_helper.hxx": cdef void raise_bout_py_error() -cdef extern from "bout/options_netcdf.hxx" namespace "bout": - cdef void writeDefaultOutputFile(); +cdef extern from "bout/options_io.hxx" namespace "bout": cdef void writeDefaultOutputFile(Options& options); - cppclass OptionsNetCDF: - enum FileMode: - replace - append - OptionsNetCDF() except +raise_bout_py_error - OptionsNetCDF(string filename) except +raise_bout_py_error - OptionsNetCDF(string filename, FileMode mode) except +raise_bout_py_error - OptionsNetCDF(const OptionsNetCDF&); - OptionsNetCDF(OptionsNetCDF&&); - OptionsNetCDF& operator=(const OptionsNetCDF&); - OptionsNetCDF& operator=(OptionsNetCDF&&); + cppclass OptionsIO: + @staticmethod + OptionsIO * create(string filename) Options read(); void write(const Options& options); void write(const Options& options, string time_dim); diff --git a/tools/pylib/_boutpp_build/boutcpp.pxd.jinja b/tools/pylib/_boutpp_build/boutcpp.pxd.jinja index c94fd14a17..12e210a5b5 100644 --- a/tools/pylib/_boutpp_build/boutcpp.pxd.jinja +++ b/tools/pylib/_boutpp_build/boutcpp.pxd.jinja @@ -5,7 +5,7 @@ from libcpp.memory cimport unique_ptr from libcpp.string cimport string cimport resolve_enum as benum -from bout_options cimport Options, OptionsReader, OptionsNetCDF, writeDefaultOutputFile +from bout_options cimport Options, OptionsReader, OptionsIO, writeDefaultOutputFile cdef extern from "boutexception_helper.hxx": cdef void raise_bout_py_error() diff --git a/tools/pylib/_boutpp_build/boutpp.pyx.jinja b/tools/pylib/_boutpp_build/boutpp.pyx.jinja index 657e2f28c1..3aeb1428eb 100644 --- a/tools/pylib/_boutpp_build/boutpp.pyx.jinja +++ b/tools/pylib/_boutpp_build/boutpp.pyx.jinja @@ -1723,7 +1723,6 @@ cdef class Options: del self.cobj self.cobj = NULL - def writeDefaultOutputFile(options: Options): c.writeDefaultOutputFile(deref(options.cobj))