[CI] update CI docker to ubuntu20.04

- Simplify the CI pipeline by removing some unnecessary testing items. - Fix the hiredis dependency to tag v1.1.0
tensorflow · Mar 11, 2023 · 64ec2e5 · 64ec2e5
1 parent 4828cf9
commit 64ec2e5
Show file tree

Hide file tree

Showing 15 changed files with 150 additions and 54 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   backport:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     name: Backport
     steps:
       - name: Backport Bot

diff --git a/.github/workflows/make_wheel_Linux_x86.sh b/.github/workflows/make_wheel_Linux_x86.sh
@@ -12,7 +12,7 @@ else
 fi
 
 if [ $TF_VERSION == "2.6.3" ]  || [ $TF_VERSION == "2.8.3" ] ; then
-  export BUILD_IMAGE="tfra/nosla-cuda11.2-cudnn8.1-ubuntu18.04-manylinux2010-multipython"
+  export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython"
   export TF_CUDA_VERSION="11.2"
   export TF_CUDNN_VERSION="8.1"
 elif [ $TF_VERSION == "2.4.1" ] ; then

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -23,7 +23,7 @@ env:
 jobs:
   test-with-bazel:
     name: Test with bazel
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
@@ -44,12 +44,12 @@ jobs:
     strategy:
       matrix:
         # TODO: add back 'windows-latest' when it can be compiled.
-        os: ['macos-12', 'ubuntu-18.04']
+        os: ['macos-12', 'ubuntu-20.04']
         py-version: ['3.7', '3.8', '3.9']
         tf-version: ['2.6.3', '2.8.3']
         tf-need-cuda: ['1', '0']
-        tf-cuda-version: ['10.0', '11.2']
-        tf-cudnn-version: ['7.6', '8.1']
+        tf-cuda-version: ['11.2']
+        tf-cudnn-version: ['8.1']
         cpu: ['x86']
 #       TODO(poinwater): add macOS CI once GitHub supports macOS 12.0.0 + 
 #        include:
@@ -62,23 +62,6 @@ jobs:
           # excludes cuda on macOS
           - os: 'macos-12'
             tf-need-cuda: '1'
-          # excludes TF2.6.3 with cuda 10.0
-          - tf-version: '2.6.3'
-            tf-cuda-version: '10.0'
-          # excludes TF2.8.3 with cuda 10.0
-          - tf-version: '2.8.3'
-            tf-cuda-version: '10.0'
-          # excludes cudnn 8 with cuda 10.0
-          - tf-cuda-version: '11.2'
-            tf-cudnn-version: '7.6'
-          # excludes cudnn 7 with cuda 11.0
-          - tf-cuda-version: '10.0'
-            tf-cudnn-version: '8.1'
-          # excludes some unnecessary step on no cuda
-          - tf-need-cuda: '0'
-            tf-cuda-version: '10.0'
-          - tf-need-cuda: '0'
-            tf-cudnn-version: '7.6'
       fail-fast: false
     steps:
       - uses: actions/[email protected]
@@ -97,7 +80,7 @@ jobs:
           python-version: ${{ matrix.py-version }}
       - name: Setup Bazel
         # Ubuntu bazel is run inside of the docker image
-        if: matrix.os != 'ubuntu-18.04'
+        if: matrix.os != 'ubuntu-20.04'
         run: bash tools/install_deps/install_bazelisk.sh ./
       - name: Build wheels
         env:
@@ -124,7 +107,7 @@ jobs:
   upload-wheels:
     name: Publish wheels to PyPi
     needs: [release-wheel, test-with-bazel]
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         # TODO: add back 'Windows' when it can be compiled.
@@ -168,7 +151,7 @@ jobs:
   upload-dev-container:
     name: Upload dev container to DockerHub
     needs: [release-wheel, test-with-bazel]
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         py-version: ['3.7', '3.8', '3.9']

diff --git a/WORKSPACE b/WORKSPACE
@@ -41,9 +41,9 @@ http_archive(
 
 new_git_repository(
     name = "hiredis",
-    branch = "master",
     build_file = "//build_deps/toolchains/redis:hiredis.BUILD",
     remote = "https://github.com/redis/hiredis.git",
+    tag = "v1.1.0",
 )
 
 http_archive(

diff --git a/...w_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py b/...w_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py
@@ -66,7 +66,7 @@ def _type_converter(tf_type):
   mapper = {
       dtypes.int32: np.int32,
       dtypes.int64: np.int64,
-      dtypes.float32: np.float,
+      dtypes.float32: float,
       dtypes.float64: np.float64,
   }
   return mapper[tf_type]
@@ -383,9 +383,8 @@ def test_higher_rank(self):
                 initializer=2.0,
                 dim=dim,
             )
-            ids = np.random.randint(2**31,
-                                    size=np.prod(ids_shape),
-                                    dtype=np.int).reshape(ids_shape)
+            ids = np.random.randint(2**31, size=np.prod(ids_shape),
+                                    dtype=int).reshape(ids_shape)
             ids = constant_op.constant(ids, dtype=dtypes.int64)
             simple = params.lookup(ids)
             self.evaluate(params.upsert(ids, simple))
@@ -414,9 +413,8 @@ def test_static_shape_checking(self):
             params_nn = variable_scope.get_variable("n",
                                                     shape=[100, dim],
                                                     use_resource=False)
-            ids = np.random.randint(2**31,
-                                    size=np.prod(ids_shape),
-                                    dtype=np.int).reshape(ids_shape)
+            ids = np.random.randint(2**31, size=np.prod(ids_shape),
+                                    dtype=int).reshape(ids_shape)
             ids = constant_op.constant(ids, dtype=dtypes.int64)
 
             embedding_test = de.embedding_lookup(params, ids)

diff --git a/...mmenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py b/...mmenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py
@@ -62,7 +62,7 @@ def _type_converter(tf_type):
   mapper = {
       dtypes.int32: np.int32,
       dtypes.int64: np.int64,
-      dtypes.float32: np.float,
+      dtypes.float32: float,
       dtypes.float64: np.float64,
   }
   return mapper[tf_type]

diff --git a/...ommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py b/...ommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py
@@ -71,13 +71,13 @@ def _type_converter(tf_type):
   mapper = {
       dtypes.int32: np.int32,
       dtypes.int64: np.int64,
-      dtypes.float32: np.float,
+      dtypes.float32: float,
       dtypes.float64: np.float64,
-      dtypes.string: np.str,
+      dtypes.string: str,
       dtypes.half: np.float16,
       dtypes.bfloat16: dtypes.bfloat16.as_numpy_dtype,
       dtypes.int8: np.int8,
-      dtypes.bool: np.bool,
+      dtypes.bool: bool,
   }
   return mapper[tf_type]
 

diff --git a/...ow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py b/...ow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py
@@ -71,12 +71,12 @@ def _type_converter(tf_type):
   mapper = {
       dtypes.int32: np.int32,
       dtypes.int64: np.int64,
-      dtypes.float32: np.float,
+      dtypes.float32: float,
       dtypes.float64: np.float64,
-      dtypes.string: np.str,
+      dtypes.string: str,
       dtypes.half: np.float16,
       dtypes.int8: np.int8,
-      dtypes.bool: np.bool,
+      dtypes.bool: bool,
   }
   return mapper[tf_type]
 

diff --git a/tools/check_python_format.py b/tools/check_python_format.py
@@ -19,7 +19,9 @@ def _run_format():
 
   if files_changed:
     print("Some files have changed.")
-    print("Please use 'yapf --style=google -ri ./**/*.py' before commit.")
+    print(
+        "Please use 'find . -name '*.py' -print0 | xargs -0 yapf --style=./.yapf -ir' before commit."
+    )
   else:
     print("No formatting needed.")
 

diff --git a/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile b/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile
@@ -0,0 +1,115 @@
+# Dockerfile to build a manylinux 2010 compliant cross-compiler.
+#
+# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
+# glibc (2.12) and system libstdc++ (4.4).
+#
+# To push a new version, run:
+# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile . \
+#  --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython"
+# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython
+
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset
+
+RUN chmod 777 /tmp/
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
+      cpio \
+      file \
+      flex \
+      g++ \
+      make \
+      patch \
+      rpm2cpio \
+      unar \
+      wget \
+      xz-utils \
+      libjpeg-dev \
+      zlib1g-dev \
+      libgflags-dev \
+      libsnappy-dev \
+      libbz2-dev \
+      liblz4-dev \
+      libzstd-dev \
+      openssh-client \
+      && \
+    rm -rf /var/lib/apt/lists/*
+
+ADD devtoolset/fixlinks.sh fixlinks.sh
+ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
+ADD devtoolset/rpm-patch.sh rpm-patch.sh
+
+# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
+RUN /build_devtoolset.sh devtoolset-7 /dt7
+# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
+RUN /build_devtoolset.sh devtoolset-8 /dt8
+
+# TODO(klimek): Split up into two different docker images.
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04
+COPY --from=devtoolset /dt7 /dt7
+COPY --from=devtoolset /dt8 /dt8
+
+# Install TensorRT.
+RUN echo \
+    deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \
+    > /etc/apt/sources.list.d/nvidia-ml.list \
+      && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \
+    apt-get update && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy and run the install scripts.
+ARG DEBIAN_FRONTEND=noninteractive
+
+COPY install/install_bootstrap_deb_packages.sh /install/
+RUN /install/install_bootstrap_deb_packages.sh
+
+COPY install/install_deb_packages.sh /install/
+RUN /install/install_deb_packages.sh
+
+# Install additional packages needed for this image:
+# - dependencies to build Python from source
+# - patchelf, as it is required by auditwheel
+RUN apt-get update && apt-get install -y \
+    libbz2-dev \
+    libffi-dev \
+    libgdbm-dev \
+    libncurses5-dev \
+    libnss3-dev \
+    libreadline-dev \
+    patchelf \
+    gcc-multilib \
+      && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN chmod 777 /tmp/
+WORKDIR /tmp/
+
+COPY install/install_nccl.sh /install/
+RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
+
+COPY install/install_rocksdb.sh /install/
+RUN /install/install_rocksdb.sh "6.22.1"
+
+COPY install/install_bazel.sh /install/
+RUN /install/install_bazel.sh "5.1.1"
+
+COPY install/build_and_install_python.sh /install/
+RUN /install/build_and_install_python.sh "3.7.7"
+RUN /install/build_and_install_python.sh "3.8.2"
+RUN /install/build_and_install_python.sh "3.9.7"
+RUN /install/build_and_install_python.sh "3.10.6"
+
+COPY install/install_pip_packages_by_version.sh /install/
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.9"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.8"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.7"
+
+COPY install/use_devtoolset_7.sh /install/
+RUN /install/use_devtoolset_7.sh
+
+COPY install/install_openmpi.sh /install/
+RUN /install/install_openmpi.sh "4.1.1"
+
+# clean
+RUN rm -rf /tmp/*
diff --git a/tools/docker/install/install_deb_packages.sh b/tools/docker/install/install_deb_packages.sh
@@ -55,7 +55,7 @@ apt-get install -y --no-install-recommends \
     pkg-config \
     python-dev \
     python-setuptools \
-    python-virtualenv \
+    python3-virtualenv \
     python3-dev \
     python3-setuptools \
     rsync \

diff --git a/tools/docker/install/install_latest_clang.sh b/tools/docker/install/install_latest_clang.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -eu
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,8 @@
 # limitations under the License.
 # ==============================================================================
 
-DIST="$(grep "DISTRIB_RELEASE" /etc/lsb-release |sed 's,.*=,,; s,\.,_,')"
-INSTALL_DIR="/clang_${CLANG_VERSION}"
-STORAGE="https://storage.googleapis.com/clang-builds-stable"
-mkdir -p "${INSTALL_DIR}"
-cd "${INSTALL_DIR}"
-wget "${STORAGE}/clang-ubuntu${DIST}/clang_${CLANG_VERSION}.tar.gz"
-tar xvzf clang_${CLANG_VERSION}.tar.gz
-rm clang_${CLANG_VERSION}.tar.gz
+DIST="$(grep "DISTRIB_CODENAME" /etc/lsb-release |sed 's,.*=,,')"
+wget -O - "https://apt.llvm.org/llvm-snapshot.gpg.key"| apt-key add -
+add-apt-repository "deb http://apt.llvm.org/${DIST}/ llvm-toolchain-${DIST}-8 main"
+apt-get update && apt-get install -y clang-8 && \
+  rm -rf /var/lib/apt/lists/*
diff --git a/tools/docker/install/install_nccl.sh b/tools/docker/install/install_nccl.sh
@@ -22,4 +22,4 @@ set -e
 cd /install/
 wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-keyring_1.0-1_all.deb
 dpkg -i cuda-keyring_1.0-1_all.deb
-apt-get update && apt-get install -y libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION}
+apt-get update && apt-get install -y  --allow-change-held-packages libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION}
diff --git a/tools/docker/install/install_pip_packages_by_version.sh b/tools/docker/install/install_pip_packages_by_version.sh
@@ -36,7 +36,7 @@ PACKAGES=(
   "werkzeug"
   "bleach"
   "markdown"
-  "protobuf"
+  "protobuf==3.20.0"
   "numpy"
   "scipy"
   "scikit-learn"

diff --git a/tools/docker/sanity_check.Dockerfile b/tools/docker/sanity_check.Dockerfile
@@ -86,6 +86,7 @@ RUN touch /ok.txt
 # Bazel code format
 FROM alpine:3.11 as check-bazel-format
 
+RUN apk update && apk add bash
 COPY ./tools/install_deps/buildifier.sh ./
 RUN sh buildifier.sh