Merge branch 'main' into chore/auto-exec-groups

.bazelci/presubmit.yml

@@ -34,6 +34,7 @@ buildifier:
   build_flags:
     - "--keep_going"
     - "--build_tag_filters=-integration-test"
+    - "--config=bazel7.x"
   test_targets:
     - "--"
     - "..."
@@ -84,16 +85,6 @@ buildifier:
     - //tests:version_3_8_test
     - //tests:version_3_9_test
     - //tests:version_default_test
-.pystar_base: &pystar_base
-  bazel: "7.x"
-  environment:
-    RULES_PYTHON_ENABLE_PYSTAR: "1"
-  build_flags:
-    - "--config=bazel7.x"
-  test_flags:
-    # The doc check tests fail because the Starlark implementation makes the
-    # PyInfo and PyRuntimeInfo symbols become documented.
-    - "--test_tag_filters=-integration-test,-doc_check_test"
 tasks:
   gazelle_extension_min:
     <<: *common_workspace_flags_min_bazel
@@ -139,26 +130,18 @@ tasks:
     name: "Default: Ubuntu, upcoming Bazel"
     platform: ubuntu2004
     bazel: last_rc
-  pystar_ubuntu_workspace:
-    <<: *reusable_config
-    <<: *pystar_base
-    name: "Default test: Ubuntu, Pystar, workspace"
-    platform: ubuntu2004
-  pystar_ubuntu_bzlmod:
+  ubuntu_workspace:
     <<: *reusable_config
-    <<: *pystar_base
-    name: "Default test: Ubuntu, Pystar, bzlmod"
+    name: "Default: Ubuntu, workspace"
     platform: ubuntu2004
-  pystar_mac_workspace:
+  mac_workspace:
     <<: *reusable_config
     <<: *common_workspace_flags
-    <<: *pystar_base
-    name: "Default test: Mac, Pystar, workspace"
+    name: "Default: Mac, workspace"
     platform: macos
-  pystar_windows_workspace:
+  windows_workspace:
     <<: *reusable_config
-    <<: *pystar_base
-    name: "Default test: Windows, Pystar, workspace"
+    name: "Default: Windows, workspace"
     platform: windows
 
   debian:
@@ -250,6 +233,13 @@ tasks:
     working_directory: examples/bzlmod
     platform: ubuntu2004
     bazel: 7.x
+  integration_test_bzlmod_ubuntu_upcoming:
+    <<: *reusable_build_test_all
+    <<: *coverage_targets_example_bzlmod
+    name: "examples/bzlmod: Ubuntu, upcoming Bazel"
+    working_directory: examples/bzlmod
+    platform: ubuntu2004
+    bazel: last_rc
   integration_test_bzlmod_debian:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
@@ -264,12 +254,27 @@ tasks:
     working_directory: examples/bzlmod
     platform: macos
     bazel: 7.x
+  integration_test_bzlmod_macos_upcoming:
+    <<: *reusable_build_test_all
+    <<: *coverage_targets_example_bzlmod
+    name: "examples/bzlmod: macOS, upcoming Bazel"
+    working_directory: examples/bzlmod
+    platform: macos
+    bazel: last_rc
   integration_test_bzlmod_windows:
     <<: *reusable_build_test_all
     # coverage is not supported on Windows
     name: "examples/bzlmod: Windows"
     working_directory: examples/bzlmod
     platform: windows
+    bazel: 7.x
+  integration_test_bzlmod_windows_upcoming:
+    <<: *reusable_build_test_all
+    # coverage is not supported on Windows
+    name: "examples/bzlmod: Windows, upcoming Bazel"
+    working_directory: examples/bzlmod
+    platform: windows
+    bazel: last_rc
   integration_test_bzlmod_ubuntu_lockfile:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

.bcr/gazelle/presubmit.yml

@@ -16,7 +16,8 @@ bcr_test_module:
   module_path: "../examples/bzlmod_build_file_generation"
   matrix:
     platform: ["debian11", "macos", "ubuntu2004", "windows"]
-    bazel: [6.x, 7.x]
+    # last_rc is to get latest 8.x release. Replace with 8.x when available.
+    bazel: [7.x, last_rc]
   tasks:
     run_tests:
       name: "Run test module"

.bcr/presubmit.yml

@@ -16,7 +16,8 @@ bcr_test_module:
   module_path: "examples/bzlmod"
   matrix:
     platform: ["debian11", "macos", "ubuntu2004", "windows"]
-    bazel: [6.x, 7.x]
+    # last_rc is to get latest 8.x release. Replace with 8.x when available.
+    bazel: [7.x, last_rc]
   tasks:
     run_tests:
       name: "Run test module"

CHANGELOG.md

@@ -30,19 +30,19 @@ Unreleased changes template.
 
 {#v0-0-0-changed}
 ### Changed
-* Nothing yet.
+* Nothing changed.
 
 {#v0-0-0-fixed}
 ### Fixed
-* Nothing yet.
+* Nothing fixed.
 
 {#v0-0-0-added}
 ### Added
-* Nothing yet.
+* Nothing added.
 
 {#v0-0-0-removed}
 ### Removed
-* Nothing yet.
+* Nothing removed.
 -->
 
 {#v0-0-0}
@@ -52,19 +52,86 @@ Unreleased changes template.
 
 {#v0-0-0-changed}
 ### Changed
-* Nothing yet.
+
+**Breaking**:
+* (toolchains) stop exposing config settings in python toolchain alias repos.
+  Please consider depending on the flags defined in
+  `//python/config_setting/...` and the `@platforms` package instead.
+* (toolchains) consumers who were depending on the `MACOS_NAME` and the `arch`
+  attribute in the `PLATFORMS` list, please update your code to respect the new
+  values. The values now correspond to the values available in the
+  `@platforms//` package constraint values.
+* (toolchains) `host_platform` and `interpreter` constants are no longer created
+  in the `toolchain` generated alias `.bzl` files. If you need to access the
+  host interpreter during the `repository_rule` evaluation, please use the
+  `@python_{version}_host//:python` targets created by
+  {bzl:obj}`python_register_toolchains` and
+  {bzl:obj}`python_register_multi_toolchains` macros or the {bzl:obj}`python`
+  bzlmod extension.
+* (bzlmod) `pip.parse.parse_all_requirements_files` attribute has been removed.
+  See notes in the previous versions about what to do.
+
+Other changes:
+* (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
+* (pypi) {bzl:obj}`pip_parse.extra_hub_aliases` now works in WORKSPACE files.
 
 {#v0-0-0-fixed}
 ### Fixed
-* Nothing yet.
+* (toolchains) stop depending on `uname` to get the value of the host platform.
+* (pypi): Correctly handle multiple versions of the same package in the requirements
+  files which is useful when including different PyTorch builds (e.g. <pytorch+cpu> vs <pytorch+cu118> ) for different target platforms.
+  Fixes ([2337](https://github.com/bazelbuild/rules_python/issues/2337)).
+* (uv): Correct the sha256sum for the `uv` binary for aarch64-apple-darwin.
+  Fixes ([2411](https://github.com/bazelbuild/rules_python/issues/2411)).
 
 {#v0-0-0-added}
 ### Added
-* Nothing yet.
+* (gazelle): Parser failures will now be logged to the terminal. Additional
+  details can be logged by setting `GAZELLE_VERBOSE=1`.
+* (toolchains) allow users to select which variant of the support host toolchain
+  they would like to use through
+  `RULES_PYTHON_REPO_TOOLCHAIN_{VERSION}_{OS}_{ARCH}` env variable setting. For
+  example, this allows one to use `freethreaded` python interpreter in the
+  `repository_rule` to build a wheel from `sdist`.
+* (toolchain) The python interpreters targeting `muslc` libc have been added
+  for the latest toolchain versions for each minor Python version. You can control
+  the toolchain selection by using the
+  {bzl:obj}`//python/config_settings:py_linux_libc` build flag.
 
 {#v0-0-0-removed}
 ### Removed
-* Nothing yet.
+* (pypi): Remove `pypi_install_dependencies` macro that has been included in
+  {bzl:obj}`py_repositories` for a long time.
+* (bzlmod): Remove `DEFAULT_PYTHON_VERSION` from `interpreters.bzl` file. If
+  you need the version, please use it from the `versions.bzl` file instead.
+
+{#v0-40-0}
+## [0.40.0] - 2024-11-17
+
+[0.40.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.40.0
+
+{#v0-40-changed}
+### Changed
+* Nothing changed.
+
+{#v0-40-fixed}
+### Fixed
+* (rules) Don't drop custom import paths if Bazel-builtin PyInfo is removed.
+  ([2414](https://github.com/bazelbuild/rules_python/issues/2414)).
+
+{#v0-40-added}
+### Added
+* Nothing added.
+
+{#v0-40-removed}
+### Removed
+* (publish) Remove deprecated `requirements.txt` for the `twine` dependencies.
+  Please use `requirements_linux.txt` instead.
+* (python_repository) Use bazel's built in `zstd` support and remove attributes
+  for customizing the `zstd` binary to be used for `zstd` archives in the
+  {bzl:obj}`python_repository` repository_rule. This affects the
+  {bzl:obj}`python_register_toolchains` and
+  {bzl:obj}`python_register_multi_toolchains` callers in the `WORKSPACE`.
 
 {#v0-39-0}
 ## [0.39.0] - 2024-11-13
@@ -149,7 +216,7 @@ Unreleased changes template.
 * (bzlmod) The extension evaluation has been adjusted to always generate the
   same lock file irrespective if `experimental_index_url` is set by any module
   or not. To opt into this behavior, set
-  {bzl:obj}`pip.parse.parse_all_requirements_files`, which will become the
+  `pip.parse.parse_all_requirements_files`, which will become the
   default in future releases leading up to `1.0.0`. Fixes
   [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
   issue is that it may break `bazel query` and in these use cases it is

MODULE.bazel

@@ -10,7 +10,7 @@ bazel_dep(name = "rules_cc", version = "0.0.14")
 bazel_dep(name = "platforms", version = "0.0.4")
 
 # Those are loaded only when using py_proto_library
-bazel_dep(name = "rules_proto", version = "6.0.2")
+bazel_dep(name = "rules_proto", version = "7.0.2")
 bazel_dep(name = "protobuf", version = "29.0-rc2", repo_name = "com_google_protobuf")
 
 internal_deps = use_extension("//python/private:internal_deps.bzl", "internal_deps")
@@ -54,7 +54,7 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
+pip = use_extension("//python/extensions:pip.bzl", "pip")
 pip.parse(
     # NOTE @aignas 2024-10-26: We have an integration test that depends on us
     # being able to build sdists for this hub, so explicitly set this to False.
@@ -84,17 +84,7 @@ bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
 bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_name = "io_bazel_rules_go")
 bazel_dep(name = "rules_python_gazelle_plugin", version = "0", dev_dependency = True)
-bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
-single_version_override(
-    module_name = "gazelle",
-    patch_strip = 1,
-    patches = [
-        # Can be removed once https://github.com/bazel-contrib/bazel-gazelle/issues/1959
-        # is fixed and released.
-        "patches/gazelle_native_sh.patch",
-    ],
-    version = "0.33.0",
-)
+bazel_dep(name = "gazelle", version = "0.40.0", dev_dependency = True, repo_name = "bazel_gazelle")
 
 internal_dev_deps = use_extension(
     "//python/private:internal_dev_deps.bzl",

WORKSPACE

@@ -86,7 +86,7 @@ load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps
 _py_gazelle_deps()
 
 # This interpreter is used for various rules_python dev-time tools
-load("@python//3.11.9:defs.bzl", "interpreter")
+interpreter = "@python_3_11_9_host//:python"
 
 #####################
 # Install twine for our own runfiles wheel publishing.
@@ -98,7 +98,7 @@ pip_parse(
     name = "rules_python_publish_deps",
     python_interpreter_target = interpreter,
     requirements_darwin = "//tools/publish:requirements_darwin.txt",
-    requirements_lock = "//tools/publish:requirements.txt",
+    requirements_lock = "//tools/publish:requirements_linux.txt",
     requirements_windows = "//tools/publish:requirements_windows.txt",
 )
 

docs/environment-variables.md

@@ -15,6 +15,16 @@ Determines the verbosity of logging output for repo rules. Valid values:
 * `TRACE`
 :::
 
+:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
+
+Determines the python interpreter platform to be used for a particular
+interpreter `(version, os, arch)` triple to be used in repository rules.
+Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
+`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
+os, arch values are the same as the ones mentioned in the
+`//python:versions.bzl` file.
+:::
+
 :::{envvar} RULES_PYTHON_PIP_ISOLATED
 
 Determines if `--isolated` is used with pip.

docs/precompiling.md

@@ -39,6 +39,30 @@ can use an opt-in or opt-out approach by setting its value:
 * targets must opt-out: `--@rules_python//python/config_settings:precompile=enabled`
 * targets must opt-in: `--@rules_python//python/config_settings:precompile=disabled`
 
+## Pyc-only builds
+
+A pyc-only build (aka "source less" builds) is when only `.pyc` files are
+included; the source `.py` files are not included.
+
+To enable this, set
+{bzl:obj}`--@rules_python//python/config_settings:precompile_source_retention=omit_source`
+flag on the command line or the {bzl:attr}`precompile_source_retention=omit_source`
+attribute on specific targets.
+
+The advantage of pyc-only builds are:
+* Fewer total files in a binary.
+* Imports _may_ be _slightly_ faster.
+
+The disadvantages are:
+* Error messages will be less precise because the precise line and offset
+  information isn't in an pyc file.
+* pyc files are Python major-version specific.
+
+:::{note}
+pyc files are not a form of hiding source code. They are trivial to uncompile,
+and uncompiling them can recover almost the original source.
+:::
+
 ## Advanced precompiler customization
 
 The default implementation of the precompiler is a persistent, multiplexed,