Buildkite gh1

.github/workflows/compute-projects-to-test/action.yml

@@ -0,0 +1,21 @@
+name: 'Compute Projects To Test'
+inputs:
+  projects:
+    required: false
+    type: 'string'
+
+outputs:
+  check-targets:
+    description: "A space delimited list of check-targets to pass to ninja."
+    value: ${{ steps.compute-projects.outputs.check-targets }}
+
+  projects:
+    description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS."
+    value: ${{ steps.compute-projects.outputs.projects }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: compute-projects
+      run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }}
+      shell: bash

.github/workflows/compute-projects-to-test/compute-projects-to-test.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This file generates a Buildkite pipeline that triggers the various CI jobs for
+# the LLVM project during pre-commit CI.
+#
+# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
+#
+# As this outputs a yaml file, it's possible to log messages to stderr or
+# prefix with "#".
+
+
+set -eu
+set -o pipefail
+
+# Environment variables script works with:
+
+# Set by GitHub
+: ${GITHUB_OUTPUT:=}
+: ${RUNNER_OS:=}
+
+# Allow users to specify which projects to build.
+all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
+if [ "$#" -ne 0 ]; then
+  wanted_projects="${@}"
+else
+  wanted_projects="${all_projects}"
+fi
+
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+echo "wanted_projects: $wanted_projects"
+
+function remove-unwanted-projects() {
+  projects=${@}
+  for project in ${projects}; do
+    if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then
+      echo "${project}"
+    fi
+  done
+}
+
+function compute-projects-to-test() {
+  projects=${@}
+  for project in ${projects}; do
+    echo "${project}"
+    case ${project} in
+    lld)
+      for p in bolt cross-project-tests; do
+        echo $p
+      done
+    ;;
+    llvm)
+      for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
+        echo $p
+      done
+    ;;
+    clang)
+      for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
+        echo $p
+      done
+    ;;
+    clang-tools-extra)
+      echo libc
+    ;;
+    mlir)
+      echo flang
+    ;;
+    *)
+      # Nothing to do
+    ;;
+    esac
+  done
+}
+
+function add-dependencies() {
+  projects=${@}
+  for project in ${projects}; do
+    echo "${project}"
+    case ${project} in
+    bolt)
+      for p in lld llvm; do
+        echo $p
+      done
+    ;;
+    cross-project-tests)
+      for p in lld clang; do
+        echo $p
+      done
+    ;;
+    clang-tools-extra)
+      for p in llvm clang; do
+        echo $p
+      done
+    ;;
+    compiler-rt|libc|openmp)
+      echo clang lld
+    ;;
+    flang|lldb)
+      for p in llvm clang; do
+        echo $p
+      done
+    ;;
+    lld|mlir|polly)
+      echo llvm
+    ;;
+    *)
+      # Nothing to do
+    ;;
+    esac
+  done
+}
+
+function exclude-linux() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    cross-project-tests) ;; # tests failing
+    lldb)                ;; # tests failing
+    openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
+    *)
+      echo "${project}"
+    ;;
+    esac
+  done
+}
+
+function exclude-windows() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    cross-project-tests) ;; # tests failing
+    compiler-rt)         ;; # tests taking too long
+    openmp)              ;; # TODO: having trouble with the Perl installation
+    libc)                ;; # no Windows support
+    lldb)                ;; # tests failing
+    bolt)                ;; # tests are not supported yet
+    *)
+      echo "${project}"
+    ;;
+    esac
+  done
+}
+
+# Prints only projects that are both present in $modified_dirs and the passed
+# list.
+function keep-modified-projects() {
+  projects=${@}
+  for project in ${projects}; do
+    if echo "$modified_dirs" | grep -q -E "^${project}$"; then
+      echo "${project}"
+    fi
+  done
+}
+
+function check-targets() {
+  projects=${@}
+  for project in ${projects}; do
+    case ${project} in
+    clang-tools-extra)
+      echo "check-clang-tools"
+    ;;
+    compiler-rt)
+      echo "check-all"
+    ;;
+    cross-project-tests)
+      echo "check-cross-project"
+    ;;
+    lldb)
+      echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
+    ;;
+    pstl)
+      echo "check-all"
+    ;;
+    libclc)
+      echo "check-all"
+    ;;
+    *)
+      echo "check-${project}"
+    ;;
+    esac
+  done
+}
+
+# Generic pipeline for projects that have not defined custom steps.
+#
+# Individual projects should instead define the pre-commit CI tests that suits their
+# needs while letting them run on the infrastructure provided by LLVM.
+
+# Figure out which projects need to be built on each platform
+modified_projects="$(keep-modified-projects ${all_projects})"
+echo "modified_projects: $modified_projects"
+
+if [ "${RUNNER_OS}" = "Linux" ]; then
+  projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
+elif [ "${RUNNER_OS}" = "Windows" ]; then
+  projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
+else
+  echo "Unknown runner OS: $RUNNER_OS"
+fi
+check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq)
+projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq)
+
+echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT
+echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT
+
+cat $GITHUB_OUTPUT

.github/workflows/pr-sccache-restore/action.yml

@@ -0,0 +1,43 @@
+name: PR sccache restore
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+      id: artifact-url
+      with:
+        script: |
+          const data = await github.rest.actions.listArtifactsForRepo({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            name: 'sccache-pr' + context.issue.number
+          })
+
+          console.log(data.data.artifacts)
+
+          for (artifact of data.data.artifacts) {
+            if (artifact.expired) {
+              continue;
+            }
+            console.log(artifact)
+            const url = await github.rest.actions.downloadArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: artifact.id,
+              archive_format: "zip"
+            })
+            console.log(url.url)
+            return url.url
+          }
+          console.log("Could not find previous sccache for this PR.")
+
+    - shell: bash
+      if: steps.artifact-url.outputs.result != ''
+      run: |
+        curl -L -o sccache-pr${{ github.event.number }}.zip ${{ steps.artifact-url.outputs.result }}
+        # Is this the best way to clear the cache?
+        rm -Rf .sccache/
+        unzip -d .sccache sccache-pr${{ github.event.number}}.zip
+        ls -ltr
+
+

.github/workflows/pr-sccache-save/action.yml

@@ -0,0 +1,40 @@
+name: PR sccache save
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+      with:
+        script: |
+          const data = await github.rest.actions.listWorkflowRunArtifacts({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            run_id: context.runId,
+            name: 'sccache-pr' + context.issue.number
+          })
+
+          console.log(data.data.artifacts)
+          if (data.data.artifacts.length == 0) {
+            return '';
+          }
+          console.log(data.data.artifacts[0])
+          const artifact_id = data.data.artifacts[0].id
+
+          // Delete the exisiting artifact so we can upload a new one with the same name.
+          github.rest.actions.deleteArtifact({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            artifact_id: artifact_id
+          })
+
+
+    - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+      with:
+        name: 'sccache-pr${{ github.event.number }}'
+        path: .sccache
+        retention-days: 7
+
+    - shell: bash
+      run: |
+        sccache --show-stats
+