Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.5 (#69)

CMakeLists.txt

@@ -46,7 +46,7 @@ endif()
 
 option(BUILD_EXAMPLES "Build examples" TRUE)
 option(BUILD_FT "Build functional tests" TRUE)
-option(BUILD_UT "Build unit tests" FALSE)
+option(BUILD_REG_TESTS "Build regression tests" TRUE)
 option(BUILD_CONFIG "Build cmake configs" TRUE)
 option(ENABLE_MPI "Enable MPI for library" TRUE)
 option(ENABLE_MPI_TESTS "Enable MPI for tests" TRUE)
@@ -70,17 +70,16 @@ message(STATUS "C compiler : ${CMAKE_C_COMPILER}")
 message(STATUS "CXX compiler : ${CMAKE_CXX_COMPILER}")
 message(STATUS "Build examples: ${BUILD_EXAMPLES}")
 message(STATUS "Build functional tests: ${BUILD_FT}")
-message(STATUS "Build unit tests: ${BUILD_UT}")
 message(STATUS "Build cmake configs: ${BUILD_CONFIG}")
-message(STATUS "Enable MPI for library: ${ENABLE_MPI}")
-message(STATUS "Enable MPI for tests: ${ENABLE_MPI_TESTS}")
-message(STATUS "Enable support for interop event functionality: ${ENABLE_SYCL_INTEROP_EVENT}")
-message(STATUS "Enable support for OFI HMEM: ${ENABLE_OFI_HMEM}")
+message(STATUS "Enable MPI support: ${ENABLE_MPI}")
+message(STATUS "Enable MPI tests support: ${ENABLE_MPI_TESTS}")
+message(STATUS "Enable SYCL interop event support: ${ENABLE_SYCL_INTEROP_EVENT}")
+message(STATUS "Enable OFI HMEM support: ${ENABLE_OFI_HMEM}")
 
 add_definitions(-DCCL_C_COMPILER="${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 add_definitions(-DCCL_CXX_COMPILER="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
-SET(MULTI_GPU_SUPPORT OFF CACHE BOOL "Enable Multi GPU extension support")
+SET(CCL_ENABLE_ZE OFF CACHE BOOL "Enable Level Zero support")
 
 set(CCL_COMMON_INSTALL_PREFIX "intel64")
 set(CMAKE_INSTALL_LIBDIR "lib")
@@ -94,11 +93,8 @@ set(CCL_INSTALL_LICENSE "${CMAKE_INSTALL_PREFIX}/licensing")
 set(CCL_INSTALL_MODULE "${CMAKE_INSTALL_PREFIX}/modulefiles")
 set(CCL_INSTALL_EXAMPLES "${CMAKE_INSTALL_PREFIX}/examples")
 set(CCL_INSTALL_TESTS "${CMAKE_INSTALL_PREFIX}/tests")
-set(CCL_INSTALL_UNIT_TESTS "${CMAKE_INSTALL_PREFIX}/tests/unit")
 set(CCL_INSTALL_KERNELS "${CMAKE_INSTALL_PREFIX}/lib/kernels")
 
-set(CCL_UNIT_TESTS_BUILD "${CMAKE_BINARY_DIR}/tests/unit")
-
 # setup dependency directories
 set(DEPS_DIR "${PROJECT_SOURCE_DIR}/deps")
 
@@ -133,25 +129,21 @@ if (${CMAKE_VERSION} VERSION_LESS 3.1)
     set(C_COMPILER_FLAGS "-std=gnu99")
 endif()
 
-# TODO: add -Wextra to c/cxx flags
-
 # common release/debug compilation settings
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_COMPILER_FLAGS} -Wall -Werror -D_GNU_SOURCE -fvisibility=internal")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_COMPILER_FLAGS} -Wall -Wextra -Wno-unused-parameter -Wno-implicit-fallthrough -Werror -D_GNU_SOURCE -fvisibility=internal")
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${C_COMPILER_FLAGS} -O0 -g -DENABLE_DEBUG")
 set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${C_COMPILER_FLAGS} -O3")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${C_COMPILER_FLAGS} -O2 -g")
 set(CMAKE_C_STANDARD 99)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMPILER_FLAGS} -Wall -Werror -D_GNU_SOURCE -fvisibility=internal")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMPILER_FLAGS} -Wall -Wextra -Wno-unused-parameter -Wno-implicit-fallthrough -Werror -D_GNU_SOURCE -fvisibility=internal")
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_COMPILER_FLAGS} -O0 -g -DENABLE_DEBUG")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CXX_COMPILER_FLAGS} -O3")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CXX_COMPILER_FLAGS} -O2 -g")
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-set(TRY_ENABLE_SYCL_L0 ON)
-
 set(COMMON_CMAKE_DIR ${PROJECT_SOURCE_DIR}/cmake)
 if (COMPUTE_BACKEND)
     message(STATUS "COMPUTE_BACKEND: ${COMPUTE_BACKEND}")
@@ -192,6 +184,7 @@ if (WITH_ASAN AND ${CMAKE_BUILD_TYPE_CASE_INSENSITIVE} STREQUAL "debug")
 endif()
 
 set_lp_env()
+set_avx_env()
 
 set(CCL_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/src)
 
@@ -227,7 +220,7 @@ file(GLOB spv_kernels "${PROJECT_SOURCE_DIR}/src/kernels/kernels.spv")
 endif()
 
 set(CCL_MAJOR_VERSION     "2021")
-set(CCL_MINOR_VERSION     "4")
+set(CCL_MINOR_VERSION     "5")
 set(CCL_UPDATE_VERSION    "0")
 set(CCL_PRODUCT_STATUS    "Gold")
 string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
@@ -257,15 +250,14 @@ if (ENABLE_MPI_TESTS)
         add_subdirectory(examples/benchmark)
         add_subdirectory(examples/common)
         add_subdirectory(examples/cpu)
-        add_subdirectory(examples/external_launcher)
+        if (BUILD_CONFIG)
+            add_subdirectory(examples/external_launcher)
+        endif()
         if (CCL_ENABLE_SYCL)
             add_subdirectory(examples/sycl)
         endif()
     endif()
     if (BUILD_FT)
         add_subdirectory(tests/functional)
     endif()
-    if (BUILD_UT AND EXISTS "${PROJECT_SOURCE_DIR}/tests/unit")
-        add_subdirectory(tests/unit)
-    endif()
 endif()

cmake/helpers.cmake

@@ -88,6 +88,38 @@ function(set_lp_env)
 
 endfunction(set_lp_env)
 
+function(set_avx_env)
+
+    set(GCC_AVX_MIN_SUPPORTED "4.9.0")
+    set(CLANG_AVX_MIN_SUPPORTED "9.0.0")
+
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "Intel"
+        OR (${CMAKE_C_COMPILER_ID} STREQUAL "Clang"
+            AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${CLANG_AVX_MIN_SUPPORTED})
+        OR (${CMAKE_C_COMPILER_ID} STREQUAL "GNU"
+            AND NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS ${GCC_AVX_MIN_SUPPORTED})
+        )
+        add_definitions(-DCCL_AVX_COMPILER)
+        set(CCL_AVX_COMPILER ON)
+    else()
+        set(CCL_AVX_COMPILER OFF)
+    endif()
+    message(STATUS "AVX compiler: ${CCL_AVX_COMPILER}")
+
+    if (CCL_AVX_COMPILER)
+        if ((${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "GNU"))
+            add_definitions(-DCCL_AVX_TARGET_ATTRIBUTES)
+            set(CCL_AVX_TARGET_ATTRIBUTES ON)
+        else()
+            set(CCL_AVX_TARGET_ATTRIBUTES OFF)
+        endif()
+        message(STATUS "AVX target attributes: ${CCL_AVX_TARGET_ATTRIBUTES}")
+    endif()
+
+    set(AVX_ENV_DEFINED 1 PARENT_SCOPE)
+
+endfunction(set_avx_env)
+
 function(check_compiler_version)
 
     set(GCC_MIN_SUPPORTED "4.8")
@@ -293,11 +325,11 @@ function(set_compute_backend COMMON_CMAKE_DIR)
     endif()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPUTE_BACKEND_FLAGS}")
     if (${COMPUTE_BACKEND_TARGET_NAME} STREQUAL "Intel::SYCL_level_zero" OR ${COMPUTE_BACKEND_TARGET_NAME} STREQUAL "ze_loader")
-        set(MULTI_GPU_SUPPORT ON PARENT_SCOPE)
-        set(MULTI_GPU_SUPPORT ON)
+        set(CCL_ENABLE_ZE ON PARENT_SCOPE)
+        set(CCL_ENABLE_ZE ON)
     endif()
-    if (MULTI_GPU_SUPPORT)
-        message(STATUS "Enable GPU support using level-zero")
+    if (CCL_ENABLE_ZE)
+        message(STATUS "Enable Level Zero support")
     endif()
 
     # need to pass these variables to overlying function

cmake/templates/oneCCLConfig.cmake.in

@@ -23,11 +23,11 @@ if (EXISTS "${CCL_CONFIGURATION}")
     set(_oneccl_subdir "${CCL_CONFIGURATION}")
 endif()
 
-if (_oneccl_subdir EQUAL "cpu_icc")
+if (_oneccl_subdir EQUAL "cpu")
     include(CheckCXXCompilerFlag)
     check_cxx_compiler_flag("-fsycl" _fsycl_option)
     if (_fsycl_option)
-        message(STATUS "STATUS: -fsycl not supported for CCL_CONFIGURATION=cpu_icc")
+        message(STATUS "STATUS: -fsycl not supported for CCL_CONFIGURATION=cpu")
     endif()
 endif()
 

deps/mpi/bin/mpigcc

@@ -99,12 +99,8 @@ fi
 # Determined by a combination of environment variables and tests within
 # configure (e.g., determining whehter -lsocket is needee)
 CC="gcc"
-MPICH_VERSION="3.3"
-CFLAGS=""
-CPPFLAGS=""
-LDFLAGS=" -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl "
-LIBS="-lm   -lpthread  -lfabric -lrt "
-MPIVERSION="2021.4"
+MPICH_VERSION="3.4a2"
+MPIVERSION="2021.5"
 MPILIBNAME="mpi"                           
 
 
@@ -594,10 +590,6 @@ fi
 final_cppflags=" "
 final_ldflags="  -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl "
 final_libs="-lpthread -lrt "
-if test "no" = "no" -o "${interlib_deps}" = "no" ; then
-    final_ldflags="${final_ldflags}  -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl -L/p/pdsd/scratch/jenkins/artefacts_impi_2019/hcoll/lib -L/p/pdsd/scratch/Uploads/IMPI/other/software/libfabric/linux/v1.9.0/lib"
-    final_libs="${final_libs}  -lm   -lpthread  -lfabric -lrt "
-fi
 
 # -----------------------------------------------------------------------
 #
@@ -622,7 +614,7 @@ if [ "$linking" = yes ] ; then
         $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} $allargs -I\"${includedir}\"
         rc=$?
     else
-        $Show $CC $CPPFLAGS $CFLAGS $allargs -I\"${includedir}\" -L\"${libdir}${MPILIBDIR}\" -L\"${libdir}\" $rpath_opt $mpilibs $I_MPI_OTHERLIBS $LDFLAGS
+        $Show $CC $CPPFLAGS $CFLAGS $allargs -I\"${includedir}\" -L\"${libdir}${MPILIBDIR}\" -L\"${libdir}\" $rpath_opt $mpilibs $I_MPI_OTHERLIBS ${final_ldflags}
         rc=$?
 
         if [ $rc -eq 0 -a "x$strip_debug_info" = "xyes" ] ; then

deps/mpi/bin/mpigxx

@@ -97,11 +97,8 @@ fi
 
 # Default settings for compiler, flags, and libraries
 CXX="g++"
-MPICH_VERSION="3.3"
-CXXFLAGS=""
-LDFLAGS=" -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl "
-LIBS="-lm   -lpthread  -lfabric -lrt "
-MPIVERSION="2021.4"
+MPICH_VERSION="3.4a2"
+MPIVERSION="2021.5"
 MPILIBNAME="mpi"
 MPICXXLIBNAME="mpicxx"
 
@@ -606,10 +603,6 @@ fi
 final_cppflags=" "
 final_ldflags="  -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl "
 final_libs="-lpthread -lrt "
-if test "no" = "no" -o "${interlib_deps}" = "no" ; then
-    final_ldflags="${final_ldflags}  -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack -Xlinker --enable-new-dtags -ldl -L/p/pdsd/scratch/jenkins/artefacts_impi_2019/hcoll/lib -L/p/pdsd/scratch/Uploads/IMPI/other/software/libfabric/linux/v1.9.0/lib"
-    final_libs="${final_libs}  -lm   -lpthread  -lfabric -lrt "
-fi
 
 # A temporary statement to invoke the compiler
 # Place the -L before any args incase there are any mpi libraries in there.
@@ -625,7 +618,7 @@ if [ "$linking" = yes ] ; then
         $Show $CXX ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} ${final_ldflags} $allargs -I\"${includedir}\"
         rc=$?
     else
-        $Show $CXX $CXXFLAGS $allargs -I\"${includedir}\" -L\"${libdir}${MPILIBDIR}\" -L\"${libdir}\" $rpath_opt $shllibpath $cxxlibs $mpilibs $I_MPI_OTHERLIBS $LDFLAGS
+        $Show $CXX $CXXFLAGS $allargs -I\"${includedir}\" -L\"${libdir}${MPILIBDIR}\" -L\"${libdir}\" $rpath_opt $shllibpath $cxxlibs $mpilibs $I_MPI_OTHERLIBS ${final_ldflags}
         rc=$?
         if [ $rc -eq 0 -a "x$strip_debug_info" = "xyes" ] ; then
             $Show objcopy --only-keep-debug ${executable} ${executable}.dbg

deps/mpi/bin/mpiicc

@@ -106,7 +106,7 @@ LDFLAGS="-ldl"
 MPILIBNAME="mpi"
 
 # MPIVERSION is the version of the MPICH2 library that mpicc is intended for
-MPIVERSION="2021.4"
+MPIVERSION="2021.5"
 #
 # Internal variables
 # Show is set to echo to cause the compilation command to be echoed instead

deps/mpi/bin/mpiicpc

@@ -107,7 +107,7 @@ MPILIBNAME="mpi"
 MPICXXLIBNAME="mpicxx"
 
 # MPIVERSION is the version of the Intel(R) MPI Library that mpiicpc is intended for
-MPIVERSION="2021.4"
+MPIVERSION="2021.5"
 
 # Internal variables
 # Show is set to echo to cause the compilation command to be echoed instead

deps/mpi/include/mpi.h

@@ -580,8 +580,8 @@ typedef int (MPI_Delete_function) ( MPI_Comm, int, void *, void * );
  * digits for REV, 1 digit for EXT and 2 digits for EXT_NUMBER. So,
  * 2019.0.0b0 will have the numeric version 20190000100.
  */
-#define I_MPI_VERSION "2021.4.0"
-#define I_MPI_NUMVERSION 20210400300
+#define I_MPI_VERSION "2021.5.0"
+#define I_MPI_NUMVERSION 20210500300
 
 /* for the datatype decoders */
 enum MPIR_Combiner_enum {