feat(musl): add musl toolchain (#2402)

With this and the previously landed #2406 users can use this in `sdist` or `whl`-only setups alike. The PR itself is very simple and just adds `musl` toolchains. Whilst at it also move the `WhlLibc` flag out of the `pypi` namespace since it is also used for `toolchain` matching now. Fixes #1211
bazelbuild · Nov 18, 2024 · e7c306d · e7c306d
1 parent 0e9f97d
commit e7c306d
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -90,6 +90,10 @@ Other changes:
   `RULES_PYTHON_REPO_TOOLCHAIN_{VERSION}_{OS}_{ARCH}` env variable setting. For
   example, this allows one to use `freethreaded` python interpreter in the
   `repository_rule` to build a wheel from `sdist`.
+* (toolchain) The python interpreters targeting `muslc` libc have been added
+  for the latest toolchain versions for each minor Python version. You can control
+  the toolchain selection by using the
+  {bzl:obj}`//python/config_settings:py_linux_libc` build flag.
 
 {#v0-0-0-removed}
 ### Removed

diff --git a/docs/toolchains.md b/docs/toolchains.md
@@ -451,3 +451,10 @@ The toolchain() calls should be in a separate BUILD file from everything else.
 This avoids Bazel having to perform unnecessary work when it discovers the list
 of available toolchains.
 :::
+
+## Toolchain selection flags
+
+Currently the following flags are used to influence toolchain selection:
+* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
+* {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting
+  the freethreaded experimental Python builds available from `3.13.0` onwards.
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
@@ -6,14 +6,14 @@ load(
     "BootstrapImplFlag",
     "ExecToolsToolchainFlag",
     "FreeThreadedFlag",
+    "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
 )
 load(
     "//python/private/pypi:flags.bzl",
     "UniversalWhlFlag",
     "UseWhlFlag",
-    "WhlLibcFlag",
     "define_pypi_internal_flags",
 )
 load(":config_settings.bzl", "construct_config_settings")
@@ -87,8 +87,8 @@ string_flag(
 # This is used for pip and hermetic toolchain resolution.
 string_flag(
     name = "py_linux_libc",
-    build_setting_default = WhlLibcFlag.GLIBC,
-    values = sorted(WhlLibcFlag.__members__.values()),
+    build_setting_default = LibcFlag.GLIBC,
+    values = LibcFlag.flag_values(),
     # NOTE: Only public because it is used in pip hub and toolchain repos.
     visibility = ["//visibility:public"],
 )

diff --git a/python/private/flags.bzl b/python/private/flags.bzl
@@ -132,3 +132,14 @@ FreeThreadedFlag = enum(
     # Do not use freethreaded python toolchain and wheels.
     NO = "no",
 )
+
+# Determines which libc flavor is preferred when selecting the toolchain and
+# linux whl distributions.
+#
+# buildifier: disable=name-conventions
+LibcFlag = FlagEnum(
+    # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
+    GLIBC = "glibc",
+    # Prefer musl wheels (e.g. musllinux_2_17_x86_64)
+    MUSL = "musl",
+)
diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl
@@ -39,7 +39,8 @@ Note, that here the specialization of musl vs manylinux wheels is the same in
 order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
 """
 
-load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag")
+load("//python/private:flags.bzl", "LibcFlag")
+load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag")
 
 FLAGS = struct(
     **{
@@ -251,14 +252,14 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
 
     elif os == "linux":
         for os_prefix, linux_libc in {
-            os: WhlLibcFlag.GLIBC,
-            "many" + os: WhlLibcFlag.GLIBC,
-            "musl" + os: WhlLibcFlag.MUSL,
+            os: LibcFlag.GLIBC,
+            "many" + os: LibcFlag.GLIBC,
+            "musl" + os: LibcFlag.MUSL,
         }.items():
-            if linux_libc == WhlLibcFlag.GLIBC:
+            if linux_libc == LibcFlag.GLIBC:
                 libc_versions = glibc_versions
                 libc_flag = FLAGS.pip_whl_glibc_version
-            elif linux_libc == WhlLibcFlag.MUSL:
+            elif linux_libc == LibcFlag.MUSL:
                 libc_versions = muslc_versions
                 libc_flag = FLAGS.pip_whl_muslc_version
             else:

diff --git a/python/private/pypi/flags.bzl b/python/private/pypi/flags.bzl
@@ -44,16 +44,6 @@ UniversalWhlFlag = enum(
     UNIVERSAL = "universal",
 )
 
-# Determines which libc flavor is preferred when selecting the linux whl distributions.
-#
-# buildifier: disable=name-conventions
-WhlLibcFlag = enum(
-    # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64)
-    GLIBC = "glibc",
-    # Prefer musl wheels (e.g. musllinux_2_17_x86_64)
-    MUSL = "musl",
-)
-
 INTERNAL_FLAGS = [
     "dist",
     "whl_plat",

diff --git a/python/versions.bzl b/python/versions.bzl
@@ -248,6 +248,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "193dc7f0284e4917d52b17a077924474882ee172872f2257cfe3375d6d468ed9",
             "x86_64-pc-windows-msvc": "5069008a237b90f6f7a86956903f2a0221b90d471daa6e4a94831eaa399e3993",
             "x86_64-unknown-linux-gnu": "c20ee831f7f46c58fa57919b75a40eb2b6a31e03fd29aaa4e8dab4b9c4b60d5d",
+            "x86_64-unknown-linux-musl": "5c1cc348e317fe7af1acd6a7f665b46eccb554b20d6533f0e76c53f44d4556cc",
         },
         "strip_prefix": "python",
     },
@@ -367,6 +368,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "90b46dfb1abd98d45663c7a2a8c45d3047a59391d8586d71b459cec7b75f662b",
             "x86_64-pc-windows-msvc": "e48952619796c66ec9719867b87be97edca791c2ef7fbf87d42c417c3331609e",
             "x86_64-unknown-linux-gnu": "3db2171e03c1a7acdc599fba583c1b92306d3788b375c9323077367af1e9d9de",
+            "x86_64-unknown-linux-musl": "ed519c47d9620eb916a6f95ec2875396e7b1a9ab993ee40b2f31b837733f318c",
         },
         "strip_prefix": "python",
     },
@@ -481,6 +483,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "1e23ffe5bc473e1323ab8f51464da62d77399afb423babf67f8e13c82b69c674",
             "x86_64-pc-windows-msvc": "647b66ff4552e70aec3bf634dd470891b4a2b291e8e8715b3bdb162f577d4c55",
             "x86_64-unknown-linux-gnu": "8b50a442b04724a24c1eebb65a36a0c0e833d35374dbdf9c9470d8a97b164cd9",
+            "x86_64-unknown-linux-musl": "d36fc77a8dd76155a7530f6235999a693b9e7c48aa11afeb5610a091cae5aa6f",
         },
         "strip_prefix": "python",
     },
@@ -559,6 +562,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "60c5271e7edc3c2ab47440b7abf4ed50fbc693880b474f74f05768f5b657045a",
             "x86_64-pc-windows-msvc": "f05531bff16fa77b53be0776587b97b466070e768e6d5920894de988bdcd547a",
             "x86_64-unknown-linux-gnu": "43576f7db1033dd57b900307f09c2e86f371152ac8a2607133afa51cbfc36064",
+            "x86_64-unknown-linux-musl": "5ed4a4078db3cbac563af66403aaa156cd6e48831d90382a1820db2b120627b5",
         },
         "strip_prefix": "python",
     },
@@ -572,6 +576,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "cff1b7e7cd26f2d47acac1ad6590e27d29829776f77e8afa067e9419f2f6ce77",
             "x86_64-pc-windows-msvc": "b25926e8ce4164cf103bacc4f4d154894ea53e07dd3fdd5ebb16fb1a82a7b1a0",
             "x86_64-unknown-linux-gnu": "2c8cb15c6a2caadaa98af51df6fe78a8155b8471cb3dd7b9836038e0d3657fb4",
+            "x86_64-unknown-linux-musl": "2f61ee3b628a56aceea63b46c7afe2df3e22a61da706606b0c8efda57f953cf4",
             "aarch64-apple-darwin-freethreaded": "efc2e71c0e05bc5bedb7a846e05f28dd26491b1744ded35ed82f8b49ccfa684b",
             "aarch64-unknown-linux-gnu-freethreaded": "59b50df9826475d24bb7eff781fa3949112b5e9c92adb29e96a09cdf1216d5bd",
             "ppc64le-unknown-linux-gnu-freethreaded": "1217efa5f4ce67fcc9f7eb64165b1bd0912b2a21bc25c1a7e2cb174a21a5df7e",
@@ -588,6 +593,7 @@ TOOL_VERSIONS = {
             "x86_64-apple-darwin": "python",
             "x86_64-pc-windows-msvc": "python",
             "x86_64-unknown-linux-gnu": "python",
+            "x86_64-unknown-linux-musl": "python",
             "aarch64-apple-darwin-freethreaded": "python/install",
             "aarch64-unknown-linux-gnu-freethreaded": "python/install",
             "ppc64le-unknown-linux-gnu-freethreaded": "python/install",
@@ -727,6 +733,17 @@ def _generate_platforms():
             # Matches the value in @platforms//cpu package
             arch = "x86_64",
         ),
+        "x86_64-unknown-linux-musl": struct(
+            compatible_with = [
+                "@platforms//os:linux",
+                "@platforms//cpu:x86_64",
+            ],
+            flag_values = {
+                libc: "musl",
+            },
+            os_name = LINUX_NAME,
+            arch = "x86_64",
+        ),
     }
 
     freethreaded = Label("//python/config_settings:py_freethreaded")