chore(BA-458): Upgrade pants to 2.23.1 and replace the scie plugin wi…

…th intrinsic pex option (#3377)

Co-authored-by: octodog <[email protected]>

.github/workflows/ci.yml

@@ -364,14 +364,20 @@ jobs:
         gha-cache-key: pants-cache-main-1-deploy-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }}
         named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }}
         cache-lmdb-store: 'false'
-    - name: Build fat packages
+    - name: Build both lazy and fat packages
       run: |
         pants --tag="scie" package '::'
-        # 'pants run' does not support parallelization
+        # Rename artifacts to have the platform suffix
+        platform_suffix="$(python scripts/get-platform-suffix.py)"
+        for artifact in dist/backendai-*; do
+          mv "${artifact}" "${artifact}-${platform_suffix}"
+        done
+        # Generate checksums. NOTE: 'pants run' does not support parallelization
         pants list --filter-tag-regex='checksum' '::' | xargs -n 1 pants run
-    - name: Clean up intermediate pex files
-      run: |
-        rm -rf dist/src.*/
+        # Merge checksums into a single file
+        cat dist/*.sha256 > dist/checksum.txt
+        sort -u dist/checksum.txt -o dist/checksum.txt
+        rm dist/*.sha256
     - name: Upload scies
       uses: actions/upload-artifact@v4
       with:

changes/3377.misc.md

@@ -0,0 +1 @@
+Upgrade pantsbuild from 2.21 to 2.23, replacing the scie plugin with the intrinsic pex's scie build support

pants.toml

@@ -1,5 +1,5 @@
 [GLOBAL]
-pants_version = "2.21.0"
+pants_version = "2.23.1"
 pythonpath = ["%(buildroot)s/tools/pants-plugins"]
 backend_packages = [
     "pants.backend.python",
@@ -12,7 +12,6 @@ backend_packages = [
     "pants.backend.plugin_development",
     "setupgen",
     "platform_resources",
-    "scie",
     "towncrier_tool",
 ]
 pants_ignore = [
@@ -84,16 +83,18 @@ setuptools = "tools/setuptools.lock"
 # first_party_depenency_version_scheme = "exact"
 
 [pex-cli]
-# Pants 2.21.0 uses Pex 2.3.1 by default.
-version = "v2.10.0"
+# Pants 2.23 uses PEX 2.16.2 by default.
+# Pin the PEX version to get benefits from a-scie/lift#116.
+version = "v2.28.1"
 known_versions = [
-    "v2.10.0|macos_arm64|de2e75c6528009051331e81e57cf05d460d0a8a3411fa9cd0b7b0ffb5d3fc23e|4170525",
-    "v2.10.0|macos_x86_64|de2e75c6528009051331e81e57cf05d460d0a8a3411fa9cd0b7b0ffb5d3fc23e|4170525",
-    "v2.10.0|linux_arm64|de2e75c6528009051331e81e57cf05d460d0a8a3411fa9cd0b7b0ffb5d3fc23e|4170525",
-    "v2.10.0|linux_x86_64|de2e75c6528009051331e81e57cf05d460d0a8a3411fa9cd0b7b0ffb5d3fc23e|4170525",
+    "v2.28.1|macos_arm64|3ba098a452db069f2524121999c784c73f0e01c8068d68133c4db83ac9c3ca5b|4366937",
+    "v2.28.1|macos_x86_64|3ba098a452db069f2524121999c784c73f0e01c8068d68133c4db83ac9c3ca5b|4366937",
+    "v2.28.1|linux_arm64|3ba098a452db069f2524121999c784c73f0e01c8068d68133c4db83ac9c3ca5b|4366937",
+    "v2.28.1|linux_x86_64|3ba098a452db069f2524121999c784c73f0e01c8068d68133c4db83ac9c3ca5b|4366937",
 ]
 # When trying a new pex version, you could find out the hash and size-in-bytes as follows:
-# $ curl -s -L https://github.com/pantsbuild/pex/releases/download/v2.1.99/pex | tee >(wc -c) >(shasum -a 256) >/dev/null
+# $ curl -s -L https://github.com/pantsbuild/pex/releases/download/v2.28.1/pex | tee >(wc -c) >(shasum -a 256) >/dev/null
+# As of PEX v2.18, the GitHub release descriptions include the checksum and size for reference.
 
 [black]
 install_from_resolve = "black"
@@ -113,8 +114,5 @@ report = ["xml", "console"]
 [mypy]
 install_from_resolve = "mypy"
 
-[towncrier]
-install_from_resolve = "towncrier"
-
 [setuptools]
 install_from_resolve = "setuptools"

scripts/get-platform-suffix.py

@@ -0,0 +1,17 @@
+#! /usr/bin/env python3
+
+import platform
+import sys
+
+arch_name_aliases = {
+    "arm64": "aarch64",  # macOS with LLVM
+    "amd64": "x86_64",  # Windows/Linux
+    "x64": "x86_64",  # Windows
+    "x32": "x86",  # Windows
+    "i686": "x86",  # Windows
+}
+
+platform_kernel = sys.platform
+platform_arch = arch_name_aliases.get(platform.machine(), platform.machine())
+
+print(f"{platform_kernel}-{platform_arch}")

src/ai/backend/account_manager/BUILD

@@ -44,33 +44,29 @@ python_distribution(
 )
 
 pex_binary(
-    name="pex",
-    entry_point="ai.backend.cli.__main__",
+    name="backendai-account-manager",
+    **common_scie_config("lazy"),
     dependencies=[
         ":src",
         ":buildscript",
         "!!stubs/trafaret:stubs",
     ],
 )
 
-scie_binary(
-    name="backendai-account-manager",
-    fat=False,
-    dependencies=[":pex"],
-    tags=["scie", "lazy"],
-)
-
-scie_binary(
+pex_binary(
     name="backendai-account-manager-fat",
-    fat=True,
-    dependencies=[":pex"],
-    tags=["scie", "fat"],
+    **common_scie_config("eager"),
+    dependencies=[
+        ":src",
+        ":buildscript",
+        "!!stubs/trafaret:stubs",
+    ],
 )
 
 run_shell_command(
     name="checksum",
     command=(
-        "find . -name 'backendai-account-manager-*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
+        "find . -name 'backendai-account-manager*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
         " {} > {}.sha256'"
     ),
     workdir="/dist",

src/ai/backend/agent/BUILD

@@ -39,6 +39,7 @@ visibility_private_component(
     ],
     allowed_dependencies=[
         "//src/ai/backend/**",
+        "!//src/ai/backend/account_manager/**",
         "!//src/ai/backend/web/**",
         "!//src/ai/backend/manager/**",
         "!//src/ai/backend/storage/**",
@@ -73,64 +74,63 @@ python_distribution(
     tags=["wheel"],
 )
 
+_common_pex_dependencies = [
+    "//src/ai/backend/kernel:src",  # included only in binary dists
+    # Bundle cuda-open and mock accelerator plugins
+    "//src/ai/backend/accelerator/cuda_open:lib",
+    "//src/ai/backend/accelerator/cuda_open:buildscript",
+    "//src/ai/backend/accelerator/mock:lib",
+    "//src/ai/backend/accelerator/mock:buildscript",
+]
+
 pex_binary(
-    name="pex",
-    entry_point="ai.backend.cli.__main__",
+    name="backendai-agent",
+    **common_scie_config("lazy"),
     dependencies=[
         ":src",
         ":buildscript",
-        "//src/ai/backend/kernel:src",  # included only in binary dists
-        # Bundle cuda-open and mock accelerator plugins
-        "//src/ai/backend/accelerator/cuda_open:lib",
-        "//src/ai/backend/accelerator/cuda_open:buildscript",
-        "//src/ai/backend/accelerator/mock:lib",
-        "//src/ai/backend/accelerator/mock:buildscript",
         "!!stubs/trafaret:stubs",
+        *_common_pex_dependencies,
         "!!./__init__.py:src-watcher",
     ],
 )
+
 pex_binary(
-    name="pex-watcher",
-    entry_point="watcher.py",
+    name="backendai-agent-fat",
+    **common_scie_config("eager"),
     dependencies=[
-        ":src-watcher",
+        ":src",
+        ":buildscript",
         "!!stubs/trafaret:stubs",
-        "!!./__init__.py:src",
+        *_common_pex_dependencies,
+        "!!./__init__.py:src-watcher",
     ],
 )
 
-scie_binary(
-    name="backendai-agent",
-    fat=False,
-    dependencies=[":pex"],
-    tags=["scie", "lazy"],
-)
-
-scie_binary(
-    name="backendai-agent-fat",
-    fat=True,
-    dependencies=[":pex"],
-    tags=["scie", "fat"],
-)
-
-scie_binary(
+pex_binary(
     name="backendai-agent-watcher",
-    fat=False,
-    dependencies=[":pex-watcher"],
-    tags=["scie", "lazy"],
+    **common_scie_config("lazy", entry_point="watcher.py"),
+    dependencies=[
+        ":src-watcher",
+        "!!stubs/trafaret:stubs",
+        "!!./__init__.py:src",
+    ],
 )
 
-scie_binary(
+pex_binary(
     name="backendai-agent-watcher-fat",
-    fat=True,
-    dependencies=[":pex-watcher"],
-    tags=["scie", "fat"],
+    **common_scie_config("eager", entry_point="watcher.py"),
+    dependencies=[
+        ":src-watcher",
+        "!!stubs/trafaret:stubs",
+        "!!./__init__.py:src",
+    ],
 )
 
 run_shell_command(
     name="checksum",
     command=(
-        "find . -name 'backendai-agent-*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
+        "find . -name 'backendai-agent*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
         " {} > {}.sha256'"
     ),
     workdir="/dist",

src/ai/backend/cli/BUILD

@@ -11,6 +11,7 @@ visibility_private_component(
     ],
     allowed_dependencies=[
         "//src/ai/backend/**",
+        "!//src/ai/backend/account_manager/**",
         "!//src/ai/backend/web/**",
         "!//src/ai/backend/manager/**",
         "!//src/ai/backend/agent/**",

src/ai/backend/client/BUILD

@@ -12,6 +12,7 @@ visibility_private_component(
     ],
     allowed_dependencies=[
         "//src/ai/backend/**",
+        "!//src/ai/backend/account_manager/**",
         "!//src/ai/backend/web/**",
         "!//src/ai/backend/manager/**",
         "!//src/ai/backend/agent/**",
@@ -41,33 +42,29 @@ python_distribution(
 )
 
 pex_binary(
-    name="pex",
-    entry_point="ai.backend.cli.__main__",
+    name="backendai-client",
+    **common_scie_config("lazy"),
     dependencies=[
         ":src",
         ":buildscript",
         "!!stubs/trafaret:stubs",
     ],
 )
 
-scie_binary(
-    name="backendai-client",
-    fat=False,
-    dependencies=[":pex"],
-    tags=["scie", "lazy"],
-)
-
-scie_binary(
+pex_binary(
     name="backendai-client-fat",
-    fat=True,
-    dependencies=[":pex"],
-    tags=["scie", "fat"],
+    **common_scie_config("eager"),
+    dependencies=[
+        ":src",
+        ":buildscript",
+        "!!stubs/trafaret:stubs",
+    ],
 )
 
 run_shell_command(
     name="checksum",
     command=(
-        "find . -name 'backendai-client-*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
+        "find . -name 'backendai-client*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
         " {} > {}.sha256'"
     ),
     workdir="/dist",

src/ai/backend/common/BUILD

@@ -12,6 +12,7 @@ visibility_private_component(
     ],
     allowed_dependencies=[
         "//src/ai/backend/**",
+        "!//src/ai/backend/account_manager/**",
         "!//src/ai/backend/web/**",
         "!//src/ai/backend/manager/**",
         "!//src/ai/backend/agent/**",

src/ai/backend/install/BUILD

@@ -12,6 +12,7 @@ visibility_private_component(
     allowed_dependencies=[
         "//src/ai/backend/**",
         "!//src/ai/backend/web/**",
+        "!//src/ai/backend/account_manager/**",
         "!//src/ai/backend/manager/**",
         "!//src/ai/backend/agent/**",
         "!//src/ai/backend/storage/**",
@@ -40,33 +41,35 @@ python_distribution(
 )
 
 pex_binary(
-    name="pex",
+    name="backendai-install",
+    extra_build_args=["--scie", "lazy"],
+    tags=["scie", "lazy"],
     entry_point="ai.backend.cli.__main__",
+    output_path="${target_name_normalized}",
     dependencies=[
         ":src",
         ":buildscript",
         "!!stubs/trafaret:stubs",
     ],
 )
 
-scie_binary(
-    name="backendai-install",
-    fat=False,
-    dependencies=[":pex"],
-    tags=["scie", "lazy"],
-)
-
-scie_binary(
+pex_binary(
     name="backendai-install-fat",
-    fat=True,
-    dependencies=[":pex"],
+    extra_build_args=["--scie", "eager"],
     tags=["scie", "fat"],
+    entry_point="ai.backend.cli.__main__",
+    output_path="${target_name_normalized}",
+    dependencies=[
+        ":src",
+        ":buildscript",
+        "!!stubs/trafaret:stubs",
+    ],
 )
 
 run_shell_command(
     name="checksum",
     command=(
-        "find . -name 'backendai-install-*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
+        "find . -name 'backendai-install*' -not -name '*.sha256' | xargs -I {} sh -c 'sha256sum"
         " {} > {}.sha256'"
     ),
     workdir="/dist",

src/ai/backend/kernel/BUILD

@@ -19,11 +19,6 @@ visibility_private_component(
     ],
 )
 
-pex_binary(
-    name="__main__",
-    entry_point="__main__.py",
-)
-
 python_distribution(
     name="dist",
     dependencies=[