release-attrpaths-superset.nix

# This expression will, as efficiently as possible, dump a
# *superset* of all attrpaths of derivations which might be
# part of a release on *any* platform.
#
# Both this expression and what ofborg uses (release-outpaths.nix)
# are essentially single-threaded (under the current cppnix
# implementation).
#
# This expression runs much, much, much faster and uses much, much
# less memory than the ofborg script by skipping the
# platform-relevance checks.  The ofborg outpaths.nix script takes
# half an hour on a 3ghz core and peaks at 60gbytes of memory; this
# expression runs on the same machine in 44 seconds with peak memory
# usage of 5gbytes.
#
# Once you have the list of attrnames you can split it up into
# $NUM_CORES batches and run the platform checks separately for each
# batch, in parallel.
#
# To dump the attrnames:
#
#   nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names
#
{ lib ? import (path + "/lib")
, trace ? false
, enableWarnings ? true
, checkMeta ? true
, path ? ./../..
}:
let

  # No release package attrpath may have any of these attrnames as
  # its initial component.
  #
  # If you can find a way to remove any of these entries without
  # causing CI to fail, please do so.
  #
  excluded-toplevel-attrs = {
    # spliced packagesets
    __splicedPackages = true;
    pkgsBuildBuild = true;
    pkgsBuildHost = true;
    pkgsBuildTarget = true;
    pkgsHostHost = true;
    pkgsHostTarget = true;
    pkgsTargetTarget = true;
    buildPackages = true;
    targetPackages = true;

    # cross packagesets
    pkgsLLVM = true;
    pkgsMusl = true;
    pkgsStatic = true;
    pkgsCross = true;
    pkgsi686Linux = true;
    pkgsLinux = true;
    pkgsExtraHardening = true;
  };

  # No release package attrname may have any of these at a component
  # anywhere in its attrpath.  These are the names of gigantic
  # top-level attrsets that have leaked into so many sub-packagesets
  # that it's easier to simply exclude them entirely.
  #
  # If you can find a way to remove any of these entries without
  # causing CI to fail, please do so.
  #
  excluded-attrnames-at-any-depth = {
    lib = true;
    override = true;
    __functor = true;
    __functionArgs = true;
    newScope = true;
    scope = true;
    pkgs = true;

    buildHaskellPackages = true;
    buildPackages = true;
    generateOptparseApplicativeCompletions = true;

    callPackage = true;
    mkDerivation = true;
    overrideDerivation = true;
    overrideScope = true;
    overrideScope' = true;

    # Special case: lib/types.nix leaks into a lot of nixos-related
    # derivations, and does not eval deeply.
    type = true;
  };

  # __attrsFailEvaluation is a temporary workaround to get top-level
  # eval to succeed (under builtins.tryEval) for the entire
  # packageset, without deep invasve changes into individual
  # packages.
  #
  # Now that CI has been added, ensuring that top-level eval will
  # not be broken by any new commits, you should not add any new
  # occurrences of __attrsFailEvaluation, and should remove them
  # wherever you are able to (doing so will likely require deep
  # adjustments within packages).  Once all of the uses of
  # __attrsFailEvaluation are removed, it will be deleted from the
  # routine below.  In the meantime,
  #
  # The intended semantics are that an attrpath rooted at pkgs is
  # part of the (unfiltered) release jobset iff all of the following
  # are true:
  #
  # 1. The first component of the attrpath is not in
  #    `excluded-toplevel-attrs`
  #
  # 2. No attrname in the attrpath belongs to the list of forbidden
  #    attrnames `excluded-attrnames-at-any-depth`
  #
  # 3. The attrpath leads to a value for which lib.isDerivation is true
  #
  # 4. No proper prefix of the attrpath has __attrsFailEvaluation=true
  #
  # 5. Any proper prefix of the attrpath at which lib.isDerivation
  #    is true also has __recurseIntoDerivationForReleaseJobs=true.
  #
  # The last condition is unfortunately necessary because there are
  # Hydra release jobnames which have proper prefixes which are
  # attrnames of derivations (!).  We should probably restructure
  # the job tree so that this is not the case.
  #
  justAttrNames = path: value:
    let
      attempt =
        if lib.isDerivation value &&
           # in some places we have *derivations* with jobsets as subattributes, ugh
           !(value.__recurseIntoDerivationForReleaseJobs or false) then
             [ path ]

        # Even wackier case: we have meta.broken==true jobs with
        # !meta.broken jobs as subattributes with license=unfree, and
        # check-meta.nix won't throw an "unfree" failure because the
        # enclosing derivation is marked broken.  Yeah.  Bonkers.
        # We should just forbid jobsets enclosed by derivations.
        else if lib.isDerivation value &&
                !value.meta.available then []

        else if !(lib.isAttrs value) then []
        else if (value.__attrsFailEvaluation or false) then []
        else lib.pipe value [
          (builtins.mapAttrs
            (name: value:
              if excluded-attrnames-at-any-depth.${name} or false then [] else
              (justAttrNames (path ++ [name]) value)))
          builtins.attrValues
          builtins.concatLists
        ];

      seq = builtins.deepSeq attempt attempt;
      tried = builtins.tryEval seq;

      result =
        if tried.success
        then tried.value
        else if enableWarnings && path != [ "AAAAAASomeThingsFailToEvaluate" ]
        then lib.warn "tryEval failed at: ${lib.concatStringsSep "." path}" []
        else [];
    in
      if !trace
      then result
      else lib.trace "** ${lib.concatStringsSep "." path}" result;

  unfiltered = import ./release-outpaths.nix {
    inherit checkMeta;
    attrNamesOnly = true;
    inherit path;
  };

  filtered = lib.pipe unfiltered [
    (pkgs: builtins.removeAttrs pkgs (builtins.attrNames excluded-toplevel-attrs))
  ];

  paths =
    [
      # I am not entirely sure why these three packages end up in
      # the Hydra jobset.  But they do, and they don't meet the
      # criteria above, so at the moment they are special-cased.
      [ "pkgsLLVM" "stdenv" ]
      [ "pkgsStatic" "stdenv" ]
      [ "pkgsMusl" "stdenv" ]
    ] ++ justAttrNames [] filtered;

  names =
    map (path: (lib.concatStringsSep "." path)) paths;

in
{
  inherit paths names;
}