From 64ec2e5c7399ec7b5b14d1b5199e4cac4d0fa669 Mon Sep 17 00:00:00 2001 From: rhdong Date: Thu, 9 Mar 2023 16:23:10 +0800 Subject: [PATCH] [CI] update CI docker to ubuntu20.04 - Simplify the CI pipeline by removing some unnecessary testing items. - Fix the hiredis dependency to tag v1.1.0 --- .github/workflows/backport.yml | 2 +- .github/workflows/make_wheel_Linux_x86.sh | 2 +- .github/workflows/release.yml | 31 ++--- WORKSPACE | 2 +- .../dynamic_embedding_ops_test.py | 12 +- .../dynamic_embedding_optimizer_test.py | 2 +- .../dynamic_embedding_variable_test.py | 6 +- .../kernel_tests/redis_table_variable_test.py | 6 +- tools/check_python_format.py | 4 +- ...20.04-manylinux2014-multipython.Dockerfile | 115 ++++++++++++++++++ tools/docker/install/install_deb_packages.sh | 2 +- tools/docker/install/install_latest_clang.sh | 15 +-- tools/docker/install/install_nccl.sh | 2 +- .../install_pip_packages_by_version.sh | 2 +- tools/docker/sanity_check.Dockerfile | 1 + 15 files changed, 150 insertions(+), 54 deletions(-) create mode 100644 tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 59dad26a0..493dc97c8 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -7,7 +7,7 @@ on: jobs: backport: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 name: Backport steps: - name: Backport Bot diff --git a/.github/workflows/make_wheel_Linux_x86.sh b/.github/workflows/make_wheel_Linux_x86.sh index c0be3a96d..8c9d95eaa 100644 --- a/.github/workflows/make_wheel_Linux_x86.sh +++ b/.github/workflows/make_wheel_Linux_x86.sh @@ -12,7 +12,7 @@ else fi if [ $TF_VERSION == "2.6.3" ] || [ $TF_VERSION == "2.8.3" ] ; then - export BUILD_IMAGE="tfra/nosla-cuda11.2-cudnn8.1-ubuntu18.04-manylinux2010-multipython" + export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython" export TF_CUDA_VERSION="11.2" export TF_CUDNN_VERSION="8.1" elif [ $TF_VERSION == "2.4.1" ] ; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7bfef6815..0755106fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ env: jobs: test-with-bazel: name: Test with bazel - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 @@ -44,12 +44,12 @@ jobs: strategy: matrix: # TODO: add back 'windows-latest' when it can be compiled. - os: ['macos-12', 'ubuntu-18.04'] + os: ['macos-12', 'ubuntu-20.04'] py-version: ['3.7', '3.8', '3.9'] tf-version: ['2.6.3', '2.8.3'] tf-need-cuda: ['1', '0'] - tf-cuda-version: ['10.0', '11.2'] - tf-cudnn-version: ['7.6', '8.1'] + tf-cuda-version: ['11.2'] + tf-cudnn-version: ['8.1'] cpu: ['x86'] # TODO(poinwater): add macOS CI once GitHub supports macOS 12.0.0 + # include: @@ -62,23 +62,6 @@ jobs: # excludes cuda on macOS - os: 'macos-12' tf-need-cuda: '1' - # excludes TF2.6.3 with cuda 10.0 - - tf-version: '2.6.3' - tf-cuda-version: '10.0' - # excludes TF2.8.3 with cuda 10.0 - - tf-version: '2.8.3' - tf-cuda-version: '10.0' - # excludes cudnn 8 with cuda 10.0 - - tf-cuda-version: '11.2' - tf-cudnn-version: '7.6' - # excludes cudnn 7 with cuda 11.0 - - tf-cuda-version: '10.0' - tf-cudnn-version: '8.1' - # excludes some unnecessary step on no cuda - - tf-need-cuda: '0' - tf-cuda-version: '10.0' - - tf-need-cuda: '0' - tf-cudnn-version: '7.6' fail-fast: false steps: - uses: actions/github-script@0.3.0 @@ -97,7 +80,7 @@ jobs: python-version: ${{ matrix.py-version }} - name: Setup Bazel # Ubuntu bazel is run inside of the docker image - if: matrix.os != 'ubuntu-18.04' + if: matrix.os != 'ubuntu-20.04' run: bash tools/install_deps/install_bazelisk.sh ./ - name: Build wheels env: @@ -124,7 +107,7 @@ jobs: upload-wheels: name: Publish wheels to PyPi needs: [release-wheel, test-with-bazel] - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: # TODO: add back 'Windows' when it can be compiled. @@ -168,7 +151,7 @@ jobs: upload-dev-container: name: Upload dev container to DockerHub needs: [release-wheel, test-with-bazel] - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: py-version: ['3.7', '3.8', '3.9'] diff --git a/WORKSPACE b/WORKSPACE index 7df12b577..4cf968237 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -41,9 +41,9 @@ http_archive( new_git_repository( name = "hiredis", - branch = "master", build_file = "//build_deps/toolchains/redis:hiredis.BUILD", remote = "https://github.com/redis/hiredis.git", + tag = "v1.1.0", ) http_archive( diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py index 1ade02b52..168594964 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py @@ -66,7 +66,7 @@ def _type_converter(tf_type): mapper = { dtypes.int32: np.int32, dtypes.int64: np.int64, - dtypes.float32: np.float, + dtypes.float32: float, dtypes.float64: np.float64, } return mapper[tf_type] @@ -383,9 +383,8 @@ def test_higher_rank(self): initializer=2.0, dim=dim, ) - ids = np.random.randint(2**31, - size=np.prod(ids_shape), - dtype=np.int).reshape(ids_shape) + ids = np.random.randint(2**31, size=np.prod(ids_shape), + dtype=int).reshape(ids_shape) ids = constant_op.constant(ids, dtype=dtypes.int64) simple = params.lookup(ids) self.evaluate(params.upsert(ids, simple)) @@ -414,9 +413,8 @@ def test_static_shape_checking(self): params_nn = variable_scope.get_variable("n", shape=[100, dim], use_resource=False) - ids = np.random.randint(2**31, - size=np.prod(ids_shape), - dtype=np.int).reshape(ids_shape) + ids = np.random.randint(2**31, size=np.prod(ids_shape), + dtype=int).reshape(ids_shape) ids = constant_op.constant(ids, dtype=dtypes.int64) embedding_test = de.embedding_lookup(params, ids) diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py index 16ea7122c..973765d71 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py @@ -62,7 +62,7 @@ def _type_converter(tf_type): mapper = { dtypes.int32: np.int32, dtypes.int64: np.int64, - dtypes.float32: np.float, + dtypes.float32: float, dtypes.float64: np.float64, } return mapper[tf_type] diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py index 61f597eba..f1af27609 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py @@ -71,13 +71,13 @@ def _type_converter(tf_type): mapper = { dtypes.int32: np.int32, dtypes.int64: np.int64, - dtypes.float32: np.float, + dtypes.float32: float, dtypes.float64: np.float64, - dtypes.string: np.str, + dtypes.string: str, dtypes.half: np.float16, dtypes.bfloat16: dtypes.bfloat16.as_numpy_dtype, dtypes.int8: np.int8, - dtypes.bool: np.bool, + dtypes.bool: bool, } return mapper[tf_type] diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py index 174f43565..0b9122a3c 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py @@ -71,12 +71,12 @@ def _type_converter(tf_type): mapper = { dtypes.int32: np.int32, dtypes.int64: np.int64, - dtypes.float32: np.float, + dtypes.float32: float, dtypes.float64: np.float64, - dtypes.string: np.str, + dtypes.string: str, dtypes.half: np.float16, dtypes.int8: np.int8, - dtypes.bool: np.bool, + dtypes.bool: bool, } return mapper[tf_type] diff --git a/tools/check_python_format.py b/tools/check_python_format.py index 2b0b7ee12..ccd7056d5 100644 --- a/tools/check_python_format.py +++ b/tools/check_python_format.py @@ -19,7 +19,9 @@ def _run_format(): if files_changed: print("Some files have changed.") - print("Please use 'yapf --style=google -ri ./**/*.py' before commit.") + print( + "Please use 'find . -name '*.py' -print0 | xargs -0 yapf --style=./.yapf -ir' before commit." + ) else: print("No formatting needed.") diff --git a/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile b/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile new file mode 100644 index 000000000..7b9bc3abd --- /dev/null +++ b/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile @@ -0,0 +1,115 @@ +# Dockerfile to build a manylinux 2010 compliant cross-compiler. +# +# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible +# glibc (2.12) and system libstdc++ (4.4). +# +# To push a new version, run: +# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile . \ +# --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython" +# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython + +FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset + +RUN chmod 777 /tmp/ +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \ + cpio \ + file \ + flex \ + g++ \ + make \ + patch \ + rpm2cpio \ + unar \ + wget \ + xz-utils \ + libjpeg-dev \ + zlib1g-dev \ + libgflags-dev \ + libsnappy-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + openssh-client \ + && \ + rm -rf /var/lib/apt/lists/* + +ADD devtoolset/fixlinks.sh fixlinks.sh +ADD devtoolset/build_devtoolset.sh build_devtoolset.sh +ADD devtoolset/rpm-patch.sh rpm-patch.sh + +# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7. +RUN /build_devtoolset.sh devtoolset-7 /dt7 +# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8. +RUN /build_devtoolset.sh devtoolset-8 /dt8 + +# TODO(klimek): Split up into two different docker images. +FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 +COPY --from=devtoolset /dt7 /dt7 +COPY --from=devtoolset /dt8 /dt8 + +# Install TensorRT. +RUN echo \ + deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \ + > /etc/apt/sources.list.d/nvidia-ml.list \ + && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \ + apt-get update && \ + rm -rf /var/lib/apt/lists/* + +# Copy and run the install scripts. +ARG DEBIAN_FRONTEND=noninteractive + +COPY install/install_bootstrap_deb_packages.sh /install/ +RUN /install/install_bootstrap_deb_packages.sh + +COPY install/install_deb_packages.sh /install/ +RUN /install/install_deb_packages.sh + +# Install additional packages needed for this image: +# - dependencies to build Python from source +# - patchelf, as it is required by auditwheel +RUN apt-get update && apt-get install -y \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + libncurses5-dev \ + libnss3-dev \ + libreadline-dev \ + patchelf \ + gcc-multilib \ + && \ + rm -rf /var/lib/apt/lists/* + +RUN chmod 777 /tmp/ +WORKDIR /tmp/ + +COPY install/install_nccl.sh /install/ +RUN /install/install_nccl.sh "2.8.4-1+cuda11.2" + +COPY install/install_rocksdb.sh /install/ +RUN /install/install_rocksdb.sh "6.22.1" + +COPY install/install_bazel.sh /install/ +RUN /install/install_bazel.sh "5.1.1" + +COPY install/build_and_install_python.sh /install/ +RUN /install/build_and_install_python.sh "3.7.7" +RUN /install/build_and_install_python.sh "3.8.2" +RUN /install/build_and_install_python.sh "3.9.7" +RUN /install/build_and_install_python.sh "3.10.6" + +COPY install/install_pip_packages_by_version.sh /install/ +RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" +RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.9" +RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.8" +RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.7" + +COPY install/use_devtoolset_7.sh /install/ +RUN /install/use_devtoolset_7.sh + +COPY install/install_openmpi.sh /install/ +RUN /install/install_openmpi.sh "4.1.1" + +# clean +RUN rm -rf /tmp/* diff --git a/tools/docker/install/install_deb_packages.sh b/tools/docker/install/install_deb_packages.sh index 9ad50e365..c5eeb28b0 100755 --- a/tools/docker/install/install_deb_packages.sh +++ b/tools/docker/install/install_deb_packages.sh @@ -55,7 +55,7 @@ apt-get install -y --no-install-recommends \ pkg-config \ python-dev \ python-setuptools \ - python-virtualenv \ + python3-virtualenv \ python3-dev \ python3-setuptools \ rsync \ diff --git a/tools/docker/install/install_latest_clang.sh b/tools/docker/install/install_latest_clang.sh index 827383e72..18eec53b4 100755 --- a/tools/docker/install/install_latest_clang.sh +++ b/tools/docker/install/install_latest_clang.sh @@ -1,5 +1,5 @@ #!/bin/bash -eu -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,11 +14,8 @@ # limitations under the License. # ============================================================================== -DIST="$(grep "DISTRIB_RELEASE" /etc/lsb-release |sed 's,.*=,,; s,\.,_,')" -INSTALL_DIR="/clang_${CLANG_VERSION}" -STORAGE="https://storage.googleapis.com/clang-builds-stable" -mkdir -p "${INSTALL_DIR}" -cd "${INSTALL_DIR}" -wget "${STORAGE}/clang-ubuntu${DIST}/clang_${CLANG_VERSION}.tar.gz" -tar xvzf clang_${CLANG_VERSION}.tar.gz -rm clang_${CLANG_VERSION}.tar.gz +DIST="$(grep "DISTRIB_CODENAME" /etc/lsb-release |sed 's,.*=,,')" +wget -O - "https://apt.llvm.org/llvm-snapshot.gpg.key"| apt-key add - +add-apt-repository "deb http://apt.llvm.org/${DIST}/ llvm-toolchain-${DIST}-8 main" +apt-get update && apt-get install -y clang-8 && \ + rm -rf /var/lib/apt/lists/* diff --git a/tools/docker/install/install_nccl.sh b/tools/docker/install/install_nccl.sh index 815ae2850..e379c225c 100755 --- a/tools/docker/install/install_nccl.sh +++ b/tools/docker/install/install_nccl.sh @@ -22,4 +22,4 @@ set -e cd /install/ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-keyring_1.0-1_all.deb dpkg -i cuda-keyring_1.0-1_all.deb -apt-get update && apt-get install -y libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION} +apt-get update && apt-get install -y --allow-change-held-packages libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION} diff --git a/tools/docker/install/install_pip_packages_by_version.sh b/tools/docker/install/install_pip_packages_by_version.sh index e4b74b48a..5259a55bb 100755 --- a/tools/docker/install/install_pip_packages_by_version.sh +++ b/tools/docker/install/install_pip_packages_by_version.sh @@ -36,7 +36,7 @@ PACKAGES=( "werkzeug" "bleach" "markdown" - "protobuf" + "protobuf==3.20.0" "numpy" "scipy" "scikit-learn" diff --git a/tools/docker/sanity_check.Dockerfile b/tools/docker/sanity_check.Dockerfile index 4c9da8fb0..d054b9713 100644 --- a/tools/docker/sanity_check.Dockerfile +++ b/tools/docker/sanity_check.Dockerfile @@ -86,6 +86,7 @@ RUN touch /ok.txt # Bazel code format FROM alpine:3.11 as check-bazel-format +RUN apk update && apk add bash COPY ./tools/install_deps/buildifier.sh ./ RUN sh buildifier.sh