Merge bitcoin#27028: ci: Cache more stuff in the ci images: msan, iwy…

…u, pip, sdks

faa0839 ci: Cache more stuff in the ci images: msan, iwyu, pip, sdks (MarcoFalke)

Pull request description:

  Now that `apt` packages are cached in the ci images, it makes sense to think about caching all other packages as well.

ACKs for top commit:
  TheCharlatan:
    re-ACK faa0839

Tree-SHA512: e2ea491570c6cdcc8522585ae7669c51ab2c0b680ff34067b58727994aa8f2e5c45ba7b76ed27a9c76d788ed155d7aade554dc164f7552fa713c00cc47b722f1

ci/test/00_setup_env.sh

@@ -65,6 +65,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
+export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
 export GOAL=${GOAL:-install}

ci/test/01_base_install.sh

@@ -6,8 +6,13 @@
 
 export LC_ALL=C.UTF-8
 
+# This script is always run as root, and the functions can be removed and
+# replaced by a bash -c "command" directly, where needed.
+# For now, they are named aliases to aid review with --color-moved=dimmed-zebra.
 CI_EXEC_ROOT () { bash -c "$*"; }
+CI_EXEC()       { bash -c "$*"; }
 export -f CI_EXEC_ROOT
+export -f CI_EXEC
 
 if [ -n "$DPKG_ADD_ARCH" ]; then
   CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH"
@@ -31,3 +36,55 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
   ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update
   ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES"
 fi
+
+if [ -n "$PIP_PACKAGES" ]; then
+  if [ "$CI_OS_NAME" == "macos" ]; then
+    sudo -H pip3 install --upgrade pip
+    # shellcheck disable=SC2086
+    IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+  else
+    # shellcheck disable=SC2086
+    ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
+  fi
+fi
+
+if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
+  CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
+  CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
+  CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
+  CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
+  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
+  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
+fi
+
+if [[ "${RUN_TIDY}" == "true" ]]; then
+  if [ ! -d "${DIR_IWYU}" ]; then
+    CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
+    CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
+    CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
+    CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
+  fi
+fi
+
+CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
+
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
+
+if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
+  OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
+  OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
+  if [ ! -f "$OSX_SDK_PATH" ]; then
+    CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
+  fi
+  CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
+fi
+
+if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
+  ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
+  if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
+    CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
+  fi
+  CI_EXEC mkdir -p "$ANDROID_HOME"
+  CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
+  CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
+fi

ci/test/04_install.sh

@@ -59,6 +59,10 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
                   --name $CONTAINER_NAME \
                   $CONTAINER_NAME)
   export CI_CONTAINER_ID
+  export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
+  export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
+  $CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}"
+  $CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ro_base/ "$BASE_ROOT_DIR"
 
   # Create a non-root user inside the container which matches the local user.
   #
@@ -67,8 +71,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER"
   docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER"
   docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}"
-  export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
-  export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
 else
   echo "Running on host system without docker wrapper"
   "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
@@ -85,17 +87,6 @@ export -f CI_EXEC_ROOT
 
 CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
 
-if [ -n "$PIP_PACKAGES" ]; then
-  if [ "$CI_OS_NAME" == "macos" ]; then
-    sudo -H pip3 install --upgrade pip
-    # shellcheck disable=SC2086
-    IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
-  else
-    # shellcheck disable=SC2086
-    ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
-  fi
-fi
-
 if [ "$CI_OS_NAME" == "macos" ]; then
   top -l 1 -s 0 | awk ' /PhysMem/ {print}'
   echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
@@ -122,30 +113,6 @@ fi
 
 CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
 
-if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
-  CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
-  CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
-  CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
-  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
-  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
-fi
-
-if [[ "${RUN_TIDY}" == "true" ]]; then
-  export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
-  if [ ! -d "${DIR_IWYU}" ]; then
-    CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
-    CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
-    CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
-    CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
-  fi
-fi
-
-if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  echo "Create $BASE_ROOT_DIR"
-  CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR"
-fi
-
 if [ "$USE_BUSY_BOX" = "true" ]; then
   echo "Setup to use BusyBox utils"
   # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)

ci/test/05_before_script.sh

@@ -14,29 +14,6 @@ else
   CI_EXEC_ROOT echo \> \$HOME/.bitcoin
 fi
 
-CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
-
-OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
-
-if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
-  OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
-  OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
-  if [ ! -f "$OSX_SDK_PATH" ]; then
-    CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
-  fi
-  CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
-fi
-
-if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
-  ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
-  if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
-    CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
-  fi
-  CI_EXEC mkdir -p "$ANDROID_HOME"
-  CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
-  CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
-fi
-
 if [ -z "$NO_DEPENDS" ]; then
   if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
     # CentOS has problems building the depends if the config shell is not explicitly set