Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support offline compilation for third-party snappy, cub and cutlass #54335

Merged
merged 12 commits into from
Jun 6, 2023
Merged

support offline compilation for third-party snappy, cub and cutlass #54335

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
289169e
update
huangjiyi Jun 4, 2023
83b69f3
update
huangjiyi Jun 4, 2023
5fa0466
update
huangjiyi Jun 4, 2023
68e78a0
update
huangjiyi Jun 4, 2023
685ea63
update tag
huangjiyi Jun 5, 2023
4ad429a
fix cutlass
huangjiyi Jun 5, 2023
4285850
update cub tag
huangjiyi Jun 5, 2023
04837f4
Merge branch 'develop' into add_submodule
huangjiyi Jun 5, 2023
7a3e60e
update
huangjiyi Jun 5, 2023
3ee1b77
update
huangjiyi Jun 5, 2023
4dac502
Merge branch 'add_submodule' of https://github.com/huangjiyi/Paddle i…
huangjiyi Jun 5, 2023
5d09ecd
Merge branch 'PaddlePaddle:develop' into add_submodule
huangjiyi Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,18 @@
path = third_party/eigen3
url = https://gitlab.com/libeigen/eigen.git
ignore = dirty
[submodule "third_party/snappy"]
path = third_party/snappy
url = https://github.com/google/snappy.git
ignore = dirty
[submodule "third_party/cub"]
path = third_party/cub
url = https://github.com/NVIDIA/cub.git
ignore = dirty
[submodule "third_party/cutlass"]
path = third_party/cutlass
url = https://github.com/NVIDIA/cutlass.git
ignore = dirty
[submodule "third_party/mkldnn"]
path = third_party/mkldnn
url = https://github.com/oneapi-src/oneDNN.git
Expand Down
11 changes: 6 additions & 5 deletions cmake/external/cub.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,27 @@ set(CUB_PATH
CACHE STRING "A path setting for external_cub path.")
set(CUB_PREFIX_DIR ${CUB_PATH})

set(CUB_REPOSITORY ${GIT_URL}/NVlabs/cub.git)
set(CUB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cub)

if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6)
# cuda_11.6/11.7/11.8‘s own cub is 1.15.0, which will cause compiling error in windows.
set(CUB_TAG 1.16.0)
execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git
--work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG})
# cub 1.16.0 is not compitable with current thrust version
add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK)
else()
set(CUB_TAG 1.8.0)
endif()
Comment on lines 27 to 36
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR 里提交的 commit 是 1.8.0 分支，所以默认下载的是 1.8.0，然后如果需要 1.16.0 分支的话会执行命令切换

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR 里提交的 commit 是 1.8.0 分支，所以默认下载的是 1.8.0，然后如果需要 1.16.0 分支的话会执行命令切换


set(CUB_INCLUDE_DIR ${CUB_PREFIX_DIR}/src/extern_cub)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个地方是否有问题？源码目录下应该没有src/extern_cub目录

Copy link
Member Author

@huangjiyi huangjiyi Jun 5, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修改，因为原来的代码 git clone 的目录是 third_party/cub/src/extern_cub，这里直接把 CUB_INCLUDE_DIR 改为
${CUB_SOURCE_DIR} 就好了

set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR}/src/extern_cub)
message("CUB_INCLUDE_DIR is ${CUB_INCLUDE_DIR}")
include_directories(${CUB_INCLUDE_DIR})

ExternalProject_Add(
extern_cub
${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
GIT_REPOSITORY ${CUB_REPOSITORY}
GIT_TAG ${CUB_TAG}
${EXTERNAL_PROJECT_LOG_ARGS}
SOURCE_DIR ${CUB_SOURCE_DIR}
PREFIX ${CUB_PREFIX_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
Expand Down
17 changes: 7 additions & 10 deletions cmake/external/cutlass.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,12 @@
include(ExternalProject)

set(CUTLASS_PREFIX_DIR ${THIRD_PARTY_PATH}/cutlass)

set(CUTLASS_REPOSITORY https://github.com/NVIDIA/cutlass.git)
set(CUTLASS_TAG v2.11.0)
set(CUTLASS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cutlass)

include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/")
include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/include/")
include_directories(
"${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/util/include/")
include_directories("${CUTLASS_SOURCE_DIR}/")
include_directories("${CUTLASS_SOURCE_DIR}/include/")
include_directories("${CUTLASS_SOURCE_DIR}/tools/util/include/")

add_definitions("-DPADDLE_WITH_CUTLASS")
add_definitions("-DSPCONV_WITH_CUTLASS=0")
Expand All @@ -33,9 +31,8 @@ endif()

ExternalProject_Add(
extern_cutlass
${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
GIT_REPOSITORY ${CUTLASS_REPOSITORY}
GIT_TAG "${CUTLASS_TAG}"
${EXTERNAL_PROJECT_LOG_ARGS}
SOURCE_DIR ${CUTLASS_SOURCE_DIR}
PREFIX ${CUTLASS_PREFIX_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
Expand All @@ -61,7 +58,7 @@ add_custom_target(
COMMAND
${PYTHON_EXECUTABLE} -B
${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
"${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
"${CUTLASS_SOURCE_DIR}/tools/library/scripts/"
"${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
"${CMAKE_CUDA_COMPILER_VERSION}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}
Expand Down
26 changes: 13 additions & 13 deletions cmake/external/snappy.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,31 @@ include(ExternalProject)

# NOTE: snappy is needed when linking with recordio

set(SNAPPY_TAG 1.1.7)
set(SNAPPY_PREFIX_DIR ${THIRD_PARTY_PATH}/snappy)
set(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
set(SNAPPY_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/snappy)
set(SNAPPY_INCLUDE_DIR
"${SNAPPY_INSTALL_DIR}/include"
"${SNAPPY_SOURCE_DIR}/include"
CACHE PATH "snappy include directory." FORCE)

if(WIN32)
set(SNAPPY_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267")
if(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
if(NOT EXISTS "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
add_custom_command(
TARGET extern_snappy
POST_BUILD
COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib
${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib)
COMMAND cmake -E copy ${SNAPPY_SOURCE_DIR}/lib/snappy.lib
${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib)
endif()
set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
else()
set(SNAPPY_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a")
set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.a")
endif()

ExternalProject_Add(
extern_snappy
GIT_REPOSITORY "https://github.com/google/snappy"
GIT_TAG "1.1.7"
SOURCE_DIR ${SNAPPY_SOURCE_DIR}
PREFIX ${SNAPPY_PREFIX_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
Expand All @@ -51,16 +51,16 @@ ExternalProject_Add(
-DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
-DCMAKE_INSTALL_PREFIX=${SNAPPY_SOURCE_DIR}
-DCMAKE_INSTALL_LIBDIR=${SNAPPY_SOURCE_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_TESTING=OFF
-DSNAPPY_BUILD_TESTS:BOOL=OFF
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
-DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_SOURCE_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_SOURCE_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
BUILD_BYPRODUCTS ${SNAPPY_LIBRARIES})
Expand Down
1 change: 1 addition & 0 deletions third_party/cub
Open in desktop
Submodule cub added at c3ccea
1 change: 1 addition & 0 deletions third_party/cutlass
Open in desktop
Submodule cutlass added at 66d9cd
1 change: 1 addition & 0 deletions third_party/snappy
Open in desktop
Submodule snappy added at b02bfa