Buildkite gh test

.github/workflows/compute-projects-to-test/action.yml

@@ -0,0 +1,21 @@
+name: 'Compute Projects To Test'
+inputs:
+  projects:
+    required: false
+    type: 'string'
+
+outputs:
+  check-targets:
+    description: "A space delimited list of check-targets to pass to ninja."
+    value: ${{ steps.compute-projects.outputs.check-targets }}
+
+  projects:
+    description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS."
+    value: ${{ steps.compute-projects.outputs.projects }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: compute-projects
+      run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }}
+      shell: bash

.github/workflows/compute-projects-to-test/compute-projects-to-test.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This file generates a Buildkite pipeline that triggers the various CI jobs for
+# the LLVM project during pre-commit CI.
+#
+# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
+#
+# As this outputs a yaml file, it's possible to log messages to stderr or
+# prefix with "#".
+
+
+set -eu
+set -o pipefail
+
+# Environment variables script works with:
+
+# Set by GitHub
+: ${GITHUB_OUTPUT:=}
+: ${RUNNER_OS:=}
+
+# Allow users to specify which projects to build.
+all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
+if [ "$#" -ne 0 ]; then
+  wanted_projects="${@}"
+else
+  wanted_projects="${all_projects}"
+fi
+
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+echo "wanted_projects: $wanted_projects"
+
+function remove-unwanted-projects() {
+  projects=${@}
+  for project in ${projects}; do
+    if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then
+      echo "${project}"
+    fi
+  done
+}
+
+function compute-projects-to-test() {
+  projects=${@}
+  for project in ${projects}; do
+    echo "${project}"
+    case ${project} in
+    lld)
+      for p in bolt cross-project-tests; do
+        echo $p
+      done
+    ;;
+    llvm)
+      for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
+        echo $p
+      done
+    ;;
+    clang)
+      for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
+        echo $p
+      done
+    ;;
+    clang-tools-extra)
+      echo libc
+    ;;
+    mlir)
+      echo flang
+    ;;
+    *)
+      # Nothing to do
+    ;;
+    esac
+  done
+}
+
+function add-dependencies() {
+  projects=${@}
+  for project in ${projects}; do
+    echo "${project}"
+    case ${project} in
+    bolt)
+      for p in lld llvm; do
+        echo $p
+      done
+    ;;
+    cross-project-tests)
+      for p in lld clang; do
+        echo $p
+      done
+    ;;
+    clang-tools-extra)
+      for p in llvm clang; do
+        echo $p
+      done
+    ;;
+    compiler-rt|libc|openmp)
+      echo clang lld
+    ;;
+    flang|lldb)
+      for p in llvm clang; do
+        echo $p
+      done
+    ;;
+    lld|mlir|polly)
+      echo llvm
+    ;;
+    *)
+      # Nothing to do
+    ;;
+    esac
+  done
+}
+
+function exclude-linux() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    cross-project-tests) ;; # tests failing
+    lldb)                ;; # tests failing
+    openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
+    *)
+      echo "${project}"
+    ;;
+    esac
+  done
+}
+
+function exclude-windows() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    cross-project-tests) ;; # tests failing
+    compiler-rt)         ;; # tests taking too long
+    openmp)              ;; # TODO: having trouble with the Perl installation
+    libc)                ;; # no Windows support
+    lldb)                ;; # tests failing
+    bolt)                ;; # tests are not supported yet
+    *)
+      echo "${project}"
+    ;;
+    esac
+  done
+}
+
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
+function keep-modified-projects() {
+  projects=${@}
+  for project in ${projects}; do
+    if echo "$modified_dirs" | grep -q -E "^${project}$"; then
+      echo "${project}"
+    fi
+  done
+}
+
+function check-targets() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    clang-tools-extra)
+      echo "check-clang-tools"
+    ;;
+    compiler-rt)
+      echo "check-all"
+    ;;
+    cross-project-tests)
+      echo "check-cross-project"
+    ;;
+    lldb)
+      echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
+    ;;
+    pstl)
+      echo "check-all"
+    ;;
+    libclc)
+      echo "check-all"
+    ;;
+    *)
+      echo "check-${project}"
+    ;;
+    esac
+  done
+}
+
+# Generic pipeline for projects that have not defined custom steps.
+#
+# Individual projects should instead define the pre-commit CI tests that suits their
+# needs while letting them run on the infrastructure provided by LLVM.
+
+# Figure out which projects need to be built on each platform
+modified_projects="$(keep-modified-projects ${all_projects})"
+echo "modified_projects: $modified_projects"
+
+if [ "${RUNNER_OS}" = "Linux" ]; then
+  projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
+elif [ "${RUNNER_OS}" = "Windows" ]; then
+  projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
+else
+  echo "Unknown runner OS: $RUNNER_OS"
+  exit 1
+fi
+check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq)
+projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq)
+
+echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT
+echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT
+
+cat $GITHUB_OUTPUT

.github/workflows/continue-timeout-job.yml

@@ -0,0 +1,70 @@
+name: Continue Timeout Job
+
+on:
+  workflow_run:
+    workflows:
+      - "Windows Precommit Tests"
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  restart:
+    name: "Restart Job"
+    permissions:
+      actions: write
+    runs-on: ubuntu-22.04
+    if: github.event.workflow_run.conclusion == 'failure'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/workflows/unprivileged-download-artifact
+          sparse-checkout-cone-mode: false
+
+      - uses: ./.github/workflows/unprivileged-download-artifact
+        id: download-artifact
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          artifact-name: timeout
+
+      - shell: bash
+        if: steps.download-artifact.outputs.filename != ''
+        run: |
+          unzip ${{ steps.download-artifact.outputs.filename }}
+
+      - name: "Restart Job"
+        if: steps.download-artifact.outputs.filename != ''
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+        with:
+          script: |
+              var fs = require('fs');
+              const data = fs.readFileSync('./timeout');
+              console.log(data);
+              const json = JSON.parse(data);
+              console.log(json);
+              if (!json || !json.job_id) {
+                console.log("Could not parse timeout artifact");
+                return;
+              }
+              const job_id = json.job_id
+
+              // Delete the timeout artifact to prepare for the next run
+              await github.rest.actions.deleteArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: ${{ steps.download-artifact.outputs.artifact-id }}
+              })
+
+              // Restart the job
+              // This function does not exist even though it is in the document
+              //github.rest.actions.reRunJobForWorkflow({
+              await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                job_id: job_id
+              })
+              console.log("Restarted job: " + job_id);
+              return "timeout"

.github/workflows/get-job-id/action.yml

@@ -0,0 +1,30 @@
+name: Get Job ID
+inputs:
+  job-name:
+    required: false
+    type: 'string'
+
+outputs:
+  job-id:
+    description: "A space delimited list of check-targets to pass to ninja."
+    value: ${{ steps.job-id.outputs.result }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+      id: job-id
+      with:
+        script: |
+          const job_data = await github.rest.actions.listJobsForWorkflowRun({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            run_id: context.runId,
+          });
+
+          for (job of job_data.data.jobs) {
+            console.log(job)
+            if (job.name == "${{ inputs.job-name }}") {
+              return job.id
+            }
+          }

.github/workflows/pr-sccache-restore/action.yml

@@ -0,0 +1,18 @@
+name: PR sccache restore
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/workflows/unprivileged-download-artifact
+      id: download-artifact
+      with:
+        run-id: ${{ github.run_id }}
+        artifact-name: sccache-pr${{ github.event.pull_request.number }}
+
+    - shell: bash
+      if: steps.download-artifact.outputs.filename != ''
+      run: |
+        # Is this the best way to clear the cache?
+        rm -Rf .sccache/
+        unzip ${{ steps.download-artifact.outputs.filename }}
+        rm ${{ steps.download-artifact.outputs.filename }}