From ae71bc8d31533492e37ed0b6d058564e2611dc66 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Tue, 3 Dec 2024 14:22:31 -0800 Subject: [PATCH] Introduce the LRE flake overlay (#1516) This change migrates the last patch to an overlay and removes the patch structure. This resolves all IFDs, cleans up the main flake and makes our custom toolchains portable across flakes. The `customClang` and `customStdenv` packages are now available as `lre.clang` and `lre.stdenv`. While the new packages are differently named, they produce the exact same setup as previously. This can be verified by the fact that `generate-toolchains` is a noop between the previous commit and this one. Since a renaming of the `customClang` executable itself would cause differences in the lre-cc toolchain we defer this to a future commit. Fixes #1262 Closes #1512 --- flake.nix | 44 +++---- local-remote-execution/lre-cc.nix | 11 +- .../overlays/clang.nix | 0 local-remote-execution/overlays/default.nix | 12 ++ local-remote-execution/overlays/stdenv.nix | 108 ++++++++++++++++++ tools/llvmStdenv.nix | 49 -------- tools/nixpkgs_link_libunwind_and_libcxx.diff | 32 ------ 7 files changed, 141 insertions(+), 115 deletions(-) rename tools/customClang.nix => local-remote-execution/overlays/clang.nix (100%) create mode 100644 local-remote-execution/overlays/default.nix create mode 100644 local-remote-execution/overlays/stdenv.nix delete mode 100644 tools/llvmStdenv.nix delete mode 100644 tools/nixpkgs_link_libunwind_and_libcxx.diff diff --git a/flake.nix b/flake.nix index 6294c2352..2738191ff 100644 --- a/flake.nix +++ b/flake.nix @@ -57,12 +57,6 @@ llvmPackages = pkgs.llvmPackages_19; - customStdenv = pkgs.callPackage ./tools/llvmStdenv.nix {inherit llvmPackages;}; - - # TODO(aaronmondal): This doesn't work with rules_rust yet. - # Tracked in https://github.com/TraceMachina/nativelink/issues/477. - customClang = pkgs.callPackage ./tools/customClang.nix {stdenv = customStdenv;}; - nixSystemToRustTriple = nixSystem: { "x86_64-linux" = "x86_64-unknown-linux-musl"; @@ -251,7 +245,7 @@ rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { inherit buildImage; - stdenv = customStdenv; + inherit (pkgs.lre) stdenv; }; createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;}; buck2-toolchain = let @@ -292,8 +286,7 @@ }; }; lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix { - inherit customClang buildImage; - stdenv = customStdenv; + inherit buildImage; }; toolchain-drake = buildImage { name = "toolchain-drake"; @@ -348,23 +341,15 @@ nativelinkCoverageForHost = nativelinkCoverageFor pkgs; in rec { - _module.args.pkgs = let - nixpkgs-patched = (import self.inputs.nixpkgs {inherit system;}).applyPatches { - name = "nixpkgs-patched"; - src = self.inputs.nixpkgs; - patches = [ - ./tools/nixpkgs_link_libunwind_and_libcxx.diff - ]; - }; - in - import nixpkgs-patched { - inherit system; - overlays = [ - (import ./tools/nixpkgs-disable-ratehammering-pulumi-tests.nix) - (import rust-overlay) - (import ./tools/rust-overlay-cut-libsecret.nix) - ]; - }; + _module.args.pkgs = import self.inputs.nixpkgs { + inherit system; + overlays = [ + self.overlays.lre + (import ./tools/nixpkgs-disable-ratehammering-pulumi-tests.nix) + (import rust-overlay) + (import ./tools/rust-overlay-cut-libsecret.nix) + ]; + }; apps = { default = { type = "app"; @@ -446,7 +431,7 @@ "/run/current-system/sw/bin" "${binutils.bintools}/bin" "${uutils-coreutils-noprefix}/bin" - "${customClang}/bin" + "${pkgs.lre.clang}/bin" "${git}/bin" "${python3}/bin" ]; @@ -497,7 +482,7 @@ # Additional tools from within our development environment. local-image-test generate-toolchains - customClang + pkgs.lre.clang native-cli docs build-chromium-tests @@ -560,5 +545,8 @@ local-remote-execution = ./local-remote-execution/flake-module.nix; nixos = ./tools/nixos/flake-module.nix; }; + overlays = { + lre = import ./local-remote-execution/overlays/default.nix; + }; }; } diff --git a/local-remote-execution/lre-cc.nix b/local-remote-execution/lre-cc.nix index 3e09fe1f7..a74ed385b 100644 --- a/local-remote-execution/lre-cc.nix +++ b/local-remote-execution/lre-cc.nix @@ -1,7 +1,6 @@ { buildImage, - customClang, - stdenv, + lre, lib, coreutils, findutils, @@ -15,15 +14,15 @@ # binary identical toolchains during local and remote execution. ("PATH=" + (lib.strings.concatStringsSep ":" [ - "${stdenv.cc.bintools}/bin" - "${customClang}/bin" - "${stdenv}/bin" + "${lre.stdenv.cc.bintools}/bin" + "${lre.clang}/bin" + "${lre.stdenv}/bin" "${coreutils}/bin" "${findutils}/bin" "${gnutar}/bin" ])) - "CC=${customClang}/bin/customClang" + "CC=${lre.clang}/bin/customClang" ]; in buildImage { diff --git a/tools/customClang.nix b/local-remote-execution/overlays/clang.nix similarity index 100% rename from tools/customClang.nix rename to local-remote-execution/overlays/clang.nix diff --git a/local-remote-execution/overlays/default.nix b/local-remote-execution/overlays/default.nix new file mode 100644 index 000000000..3595a96fd --- /dev/null +++ b/local-remote-execution/overlays/default.nix @@ -0,0 +1,12 @@ +final: _prev: { + lre = { + stdenv = final.callPackage ./stdenv.nix { + llvmPackages = final.llvmPackages_19; + targetPackages = final; + }; + + clang = final.callPackage ./clang.nix { + inherit (final.lre) stdenv; + }; + }; +} diff --git a/local-remote-execution/overlays/stdenv.nix b/local-remote-execution/overlays/stdenv.nix new file mode 100644 index 000000000..918d0640c --- /dev/null +++ b/local-remote-execution/overlays/stdenv.nix @@ -0,0 +1,108 @@ +{ + lib, + llvmPackages, + overrideCC, + stdenv, + targetPackages, + useMoldLinker, + wrapCCWith, +}: let + # Adapted from clangUseLLVM. + # See: https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/compilers/llvm/common/default.nix + clangVersion = "19"; + targetLlvmLibraries = targetPackages.llvmPackages_19.libraries; + + mkExtraBuildCommands0 = cc: '' + rsrc="$out/resource-root" + mkdir "$rsrc" + ln -s "${lib.getLib cc}/lib/clang/${clangVersion}/include" "$rsrc" + echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags + ''; + mkExtraBuildCommands = cc: + mkExtraBuildCommands0 cc + + '' + ln -s "${targetLlvmLibraries.compiler-rt.out}/lib" "$rsrc/lib" + ln -s "${targetLlvmLibraries.compiler-rt.out}/share" "$rsrc/share" + ''; + + toolchain' = wrapCCWith ( + rec { + cc = llvmPackages.clang-unwrapped; + inherit (targetLlvmLibraries) libcxx; + inherit (llvmPackages) bintools; + extraPackages = [ + targetLlvmLibraries.compiler-rt + targetLlvmLibraries.libunwind + ]; + extraBuildCommands = mkExtraBuildCommands cc; + } + // { + nixSupport.cc-cflags = + [ + "-rtlib=compiler-rt" + "-Wno-unused-command-line-argument" + "-B${targetLlvmLibraries.compiler-rt}/lib" + "-stdlib=libc++" + ] + ++ lib.optional stdenv.targetPlatform.isLinux "-fuse-ld=mold" + ++ lib.optional ( + !stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD + ) "--unwindlib=libunwind" + ++ lib.optional ( + !stdenv.targetPlatform.isWasm + && !stdenv.targetPlatform.isFreeBSD + && stdenv.targetPlatform.useLLVM or false + ) ["-lunwind" "-lc++"] + ++ lib.optional stdenv.targetPlatform.isWasm "-fno-exceptions"; + nixSupport.cc-ldflags = + lib.optionals ( + !stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD + ) [ + "-L${targetLlvmLibraries.libunwind}/lib" + "-rpath" + "${targetLlvmLibraries.libunwind}/lib" + "-L${targetLlvmLibraries.libcxx}/lib" + "-rpath" + "${targetLlvmLibraries.libcxx}/lib" + ]; + } + ); + + stdenv' = + overrideCC + llvmPackages.libcxxStdenv + toolchain'; + + toolchain = + if stdenv.targetPlatform.isDarwin + then stdenv' # Mold doesn't support darwin. + else useMoldLinker stdenv'; +in + # This toolchain uses Clang as compiler, Mold as linker, libc++ as C++ + # standard library and compiler-rt as compiler runtime. Resulting rust + # binaries depend dynamically linked on the nixpkgs distribution of glibc. + # C++ binaries additionally depend dynamically on libc++, libunwind and + # libcompiler-rt. Due to a bug we also depend on libgcc_s. + # + # TODO(aaronmondal): At the moment this toolchain is only used for the Cargo + # build. The Bazel build uses a different mostly hermetic LLVM toolchain. We + # should merge the two by generating the Bazel cc_toolchain from this stdenv. + # This likely requires a rewrite of + # https://github.com/bazelbuild/bazel-toolchains as the current implementation + # has poor compatibility with custom container images and doesn't support + # generating toolchain configs from image archives. + # + # TODO(aaronmondal): Due to various issues in the nixpkgs LLVM toolchains + # we're not getting a pure Clang/LLVM toolchain here. My guess is that the + # runtimes were not built with the degenerate LLVM toolchain but with the + # regular GCC stdenv from nixpkgs. + # + # For instance, outputs depend on libgcc_s since libcxx seems to have been was + # built with a GCC toolchain. We're also not using builtin atomics, or at + # least we're redundantly linking libatomic. + # + # Fix this as it fixes a large number of issues, including better + # cross-platform compatibility, reduced closure size, and + # static-linking-friendly licensing. This requires building the llvm project + # with the correct multistage bootstrapping process. + toolchain diff --git a/tools/llvmStdenv.nix b/tools/llvmStdenv.nix deleted file mode 100644 index 963ee827f..000000000 --- a/tools/llvmStdenv.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ - stdenv, - overrideCC, - useMoldLinker, - llvmPackages, -}: let - llvmPackagesNoBintools = llvmPackages.override { - bootBintools = null; - bootBintoolsNoLibc = null; - }; - - stdenv' = - overrideCC - llvmPackagesNoBintools.libcxxStdenv - llvmPackagesNoBintools.clangUseLLVM; - - toolchain = - if stdenv.targetPlatform.isDarwin - then stdenv' # Mold doesn't support darwin. - else useMoldLinker stdenv'; -in - # This toolchain uses Clang as compiler, Mold as linker, libc++ as C++ - # standard library and compiler-rt as compiler runtime. Resulting rust - # binaries depend dynamically linked on the nixpkgs distribution of glibc. - # C++ binaries additionally depend dynamically on libc++, libunwind and - # libcompiler-rt. Due to a bug we also depend on libgcc_s. - # - # TODO(aaronmondal): At the moment this toolchain is only used for the Cargo - # build. The Bazel build uses a different mostly hermetic LLVM toolchain. We - # should merge the two by generating the Bazel cc_toolchain from this stdenv. - # This likely requires a rewrite of - # https://github.com/bazelbuild/bazel-toolchains as the current implementation - # has poor compatibility with custom container images and doesn't support - # generating toolchain configs from image archives. - # - # TODO(aaronmondal): Due to various issues in the nixpkgs LLVM toolchains - # we're not getting a pure Clang/LLVM toolchain here. My guess is that the - # runtimes were not built with the degenerate LLVM toolchain but with the - # regular GCC stdenv from nixpkgs. - # - # For instance, outputs depend on libgcc_s since libcxx seems to have been was - # built with a GCC toolchain. We're also not using builtin atomics, or at - # least we're redundantly linking libatomic. - # - # Fix this as it fixes a large number of issues, including better - # cross-platform compatibility, reduced closure size, and - # static-linking-friendly licensing. This requires building the llvm project - # with the correct multistage bootstrapping process. - toolchain diff --git a/tools/nixpkgs_link_libunwind_and_libcxx.diff b/tools/nixpkgs_link_libunwind_and_libcxx.diff deleted file mode 100644 index 4ce44b416..000000000 --- a/tools/nixpkgs_link_libunwind_and_libcxx.diff +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/pkgs/development/compilers/llvm/common/default.nix b/pkgs/development/compilers/llvm/common/default.nix -index 707bb91b875d..d4b4fb2b5427 100644 ---- a/pkgs/development/compilers/llvm/common/default.nix -+++ b/pkgs/development/compilers/llvm/common/default.nix -@@ -780,7 +780,9 @@ let - "-rtlib=compiler-rt" - "-Wno-unused-command-line-argument" - "-B${targetLlvmLibraries.compiler-rt}/lib" -+ "-stdlib=libc++" - ] -+ ++ lib.optional stdenv.targetPlatform.isLinux "-fuse-ld=mold" - ++ lib.optional ( - !stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD - ) "--unwindlib=libunwind" -@@ -788,11 +790,16 @@ let - !stdenv.targetPlatform.isWasm - && !stdenv.targetPlatform.isFreeBSD - && stdenv.targetPlatform.useLLVM or false -- ) "-lunwind" -+ ) ["-lunwind" "-lc++"] - ++ lib.optional stdenv.targetPlatform.isWasm "-fno-exceptions"; - nixSupport.cc-ldflags = lib.optionals ( - !stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD -- ) [ "-L${targetLlvmLibraries.libunwind}/lib" ]; -+ ) [ -+ "-L${targetLlvmLibraries.libunwind}/lib" -+ "-rpath" "${targetLlvmLibraries.libunwind}/lib" -+ "-L${targetLlvmLibraries.libcxx}/lib" -+ "-rpath" "${targetLlvmLibraries.libcxx}/lib" -+ ]; - } - );