diff --git a/.github/workflows/compute-projects-to-test/action.yml b/.github/workflows/compute-projects-to-test/action.yml new file mode 100644 index 00000000000000..37df06c8c301c5 --- /dev/null +++ b/.github/workflows/compute-projects-to-test/action.yml @@ -0,0 +1,21 @@ +name: 'Compute Projects To Test' +inputs: + projects: + required: false + type: 'string' + +outputs: + check-targets: + description: "A space delimited list of check-targets to pass to ninja." + value: ${{ steps.compute-projects.outputs.check-targets }} + + projects: + description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS." + value: ${{ steps.compute-projects.outputs.projects }} + +runs: + using: "composite" + steps: + - id: compute-projects + run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }} + shell: bash diff --git a/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh new file mode 100755 index 00000000000000..807142668f618b --- /dev/null +++ b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This file generates a Buildkite pipeline that triggers the various CI jobs for +# the LLVM project during pre-commit CI. +# +# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. +# +# As this outputs a yaml file, it's possible to log messages to stderr or +# prefix with "#". + + +set -eu +set -o pipefail + +# Environment variables script works with: + +# Set by GitHub +: ${GITHUB_OUTPUT:=} +: ${RUNNER_OS:=} + +# Allow users to specify which projects to build. +all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" +if [ "$#" -ne 0 ]; then + wanted_projects="${@}" +else + wanted_projects="${all_projects}" +fi + +# List of files affected by this commit +: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)} + +echo "Files modified:" >&2 +echo "$MODIFIED_FILES" >&2 +modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u) +echo "Directories modified:" >&2 +echo "$modified_dirs" >&2 +echo "wanted_projects: $wanted_projects" + +function remove-unwanted-projects() { + projects=${@} + for project in ${projects}; do + if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function compute-projects-to-test() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + lld) + for p in bolt cross-project-tests; do + echo $p + done + ;; + llvm) + for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do + echo $p + done + ;; + clang) + for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do + echo $p + done + ;; + clang-tools-extra) + echo libc + ;; + mlir) + echo flang + ;; + *) + # Nothing to do + ;; + esac + done +} + +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + for p in lld llvm; do + echo $p + done + ;; + cross-project-tests) + for p in lld clang; do + echo $p + done + ;; + clang-tools-extra) + for p in llvm clang; do + echo $p + done + ;; + compiler-rt|libc|openmp) + echo clang lld + ;; + flang|lldb) + for p in llvm clang; do + echo $p + done + ;; + lld|mlir|polly) + echo llvm + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +# Prints only projects that are both present in $modified_dirs and the passed +# list. +function keep-modified-projects() { + projects=${@} + for project in ${projects}; do + if echo "$modified_dirs" | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + libclc) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +# Generic pipeline for projects that have not defined custom steps. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. + +# Figure out which projects need to be built on each platform +modified_projects="$(keep-modified-projects ${all_projects})" +echo "modified_projects: $modified_projects" + +if [ "${RUNNER_OS}" = "Linux" ]; then + projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects})) +elif [ "${RUNNER_OS}" = "Windows" ]; then + projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects})) +else + echo "Unknown runner OS: $RUNNER_OS" +fi +check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq) +projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq) + +echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT +echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT + +cat $GITHUB_OUTPUT diff --git a/.github/workflows/pr-sccache-restore/action.yml b/.github/workflows/pr-sccache-restore/action.yml new file mode 100644 index 00000000000000..abae90f7929c84 --- /dev/null +++ b/.github/workflows/pr-sccache-restore/action.yml @@ -0,0 +1,43 @@ +name: PR sccache restore + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + id: artifact-url + with: + script: | + const data = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'sccache-pr' + context.issue.number + }) + + console.log(data.data.artifacts) + + for (artifact of data.data.artifacts) { + if (artifact.expired) { + continue; + } + console.log(artifact) + const url = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip" + }) + console.log(url.url) + return url.url + } + console.log("Could not find previous sccache for this PR.") + + - shell: bash + if: steps.artifact-url.outputs.result != '' + run: | + curl -L -o sccache-pr${{ github.event.number }}.zip ${{ steps.artifact-url.outputs.result }} + # Is this the best way to clear the cache? + rm -Rf .sccache/ + unzip -d .sccache sccache-pr${{ github.event.number}}.zip + ls -ltr + + diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml new file mode 100644 index 00000000000000..def50fbc14d537 --- /dev/null +++ b/.github/workflows/pr-sccache-save/action.yml @@ -0,0 +1,40 @@ +name: PR sccache save + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + with: + script: | + const data = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + name: 'sccache-pr' + context.issue.number + }) + + console.log(data.data.artifacts) + if (data.data.artifacts.length == 0) { + return ''; + } + console.log(data.data.artifacts[0]) + const artifact_id = data.data.artifacts[0].id + + // Delete the exisiting artifact so we can upload a new one with the same name. + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact_id + }) + + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + with: + name: 'sccache-pr${{ github.event.number }}' + path: .sccache + retention-days: 7 + + - shell: bash + run: | + sccache --show-stats + diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml new file mode 100644 index 00000000000000..a3a03a3de38390 --- /dev/null +++ b/.github/workflows/precommit-windows.yml @@ -0,0 +1,137 @@ +name: "Windows Precommit Tests" + +permissions: + contents: read + actions: write + +on: + pull_request: + branches: + - main + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + compute-projects: + name: "Compute Projects to Test" + runs-on: ubuntu-22.04 + outputs: + projects: ${{ steps.vars.outputs.projects }} + check-targets: ${{ steps.vars.outputs.check-targets }} + test-build: ${{ steps.vars.outputs.check-targets != '' }} + steps: + - name: Fetch LLVM sources + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Compute projects to test + id: vars + uses: ./.github/workflows/compute-projects-to-test + + build-windows: + name: "Build" + runs-on: windows-2022 + permissions: + actions: write #pr-sccache-save may delete artifacts. + outputs: + build-timeout: ${{ steps.build.outputs.timeout }} + build-failed: ${{ steps.build.outputs.failed }} + needs: + - compute-projects + if: ${{ needs.compute-projects.outputs.test-build == 'true' }} + steps: + - name: Download Artifact + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + pattern: timeout-build + merge-multiple: true + + - name: Unpack Artifact + id: timeout-artifact + shell: bash + run: | + if [ -e llvm-project.tar.zst ]; then + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Setup Windows + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + + - name: Fetch LLVM sources + uses: actions/checkout@v4 + if: ${{ steps.timeout-artifact.outputs.exists != 'true' }} + + - name: Setup sccache + uses: hendrikmuhs/ccache-action@v1 + with: + max-size: 2G + variant: sccache + key: precommit-windows + + - name: Restore sccache from previous PR run + uses: ./.github/workflows/pr-sccache-restore + + - name: Configure + shell: bash + run: | + cmake -B build -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="${{ needs.compute-projects.outputs.projects }}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_LIT_ARGS="-v --no-progress-bar" \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -S llvm + + - name: Build + shell: bash + id: build + timeout-minutes: 330 + run: | + touch timeout + ninja -C build -k 0 ${{ needs.compute-projects.outputs.check-targets }} && pass=1 + rm timeout + [ $pass ] || false + + - name: Upload Timeout Message + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + id: timeout + if: always() + with: + name: timeout + path: timeout + retention-days: 2 + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save + + - name: Package Build Directory + shell: bash + if: always() && steps.timeout.outputs.artifact-id != '' + run: | + # Remove the timeout file, so the next build isn't categorized + # as a timeout. + rm -f timeout + # Dereference symlinks so that this works on Windows. + tar -h -c . | zstd -T0 -c > ../llvm-project.tar.zst + mv ../llvm-project.tar.zst . + + - name: Upload Build + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() && steps.timeout.outputs.artifact-id != '' + with: + name: timeout-build + path: llvm-project.tar.zst + retention-days: 2 diff --git a/llvm/a b/llvm/a new file mode 100644 index 00000000000000..e69de29bb2d1d6