From 3093657b77363c0f1066cb9aaf76623c2ef32286 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 16 Feb 2024 21:34:02 +0000 Subject: [PATCH 01/38] [workflows] Port buildkite Windows config to GitHub actions This reuses most of the generate-buildkite-pipeline-premerge script which determines which projects to build and which check targets to use. This new workflow only tests clang, llvm, and lld due to resource contraints on the GitHub runners. --- .../compute-projects-to-test/action.yml | 21 ++ .../compute-projects-to-test.sh | 220 ++++++++++++++++++ .github/workflows/precommit-windows.yml | 61 +++++ 3 files changed, 302 insertions(+) create mode 100644 .github/workflows/compute-projects-to-test/action.yml create mode 100755 .github/workflows/compute-projects-to-test/compute-projects-to-test.sh create mode 100644 .github/workflows/precommit-windows.yml diff --git a/.github/workflows/compute-projects-to-test/action.yml b/.github/workflows/compute-projects-to-test/action.yml new file mode 100644 index 00000000000000..37df06c8c301c5 --- /dev/null +++ b/.github/workflows/compute-projects-to-test/action.yml @@ -0,0 +1,21 @@ +name: 'Compute Projects To Test' +inputs: + projects: + required: false + type: 'string' + +outputs: + check-targets: + description: "A space delimited list of check-targets to pass to ninja." + value: ${{ steps.compute-projects.outputs.check-targets }} + + projects: + description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS." + value: ${{ steps.compute-projects.outputs.projects }} + +runs: + using: "composite" + steps: + - id: compute-projects + run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }} + shell: bash diff --git a/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh new file mode 100755 index 00000000000000..807142668f618b --- /dev/null +++ b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This file generates a Buildkite pipeline that triggers the various CI jobs for +# the LLVM project during pre-commit CI. +# +# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. +# +# As this outputs a yaml file, it's possible to log messages to stderr or +# prefix with "#". + + +set -eu +set -o pipefail + +# Environment variables script works with: + +# Set by GitHub +: ${GITHUB_OUTPUT:=} +: ${RUNNER_OS:=} + +# Allow users to specify which projects to build. +all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" +if [ "$#" -ne 0 ]; then + wanted_projects="${@}" +else + wanted_projects="${all_projects}" +fi + +# List of files affected by this commit +: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)} + +echo "Files modified:" >&2 +echo "$MODIFIED_FILES" >&2 +modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u) +echo "Directories modified:" >&2 +echo "$modified_dirs" >&2 +echo "wanted_projects: $wanted_projects" + +function remove-unwanted-projects() { + projects=${@} + for project in ${projects}; do + if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function compute-projects-to-test() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + lld) + for p in bolt cross-project-tests; do + echo $p + done + ;; + llvm) + for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do + echo $p + done + ;; + clang) + for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do + echo $p + done + ;; + clang-tools-extra) + echo libc + ;; + mlir) + echo flang + ;; + *) + # Nothing to do + ;; + esac + done +} + +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + for p in lld llvm; do + echo $p + done + ;; + cross-project-tests) + for p in lld clang; do + echo $p + done + ;; + clang-tools-extra) + for p in llvm clang; do + echo $p + done + ;; + compiler-rt|libc|openmp) + echo clang lld + ;; + flang|lldb) + for p in llvm clang; do + echo $p + done + ;; + lld|mlir|polly) + echo llvm + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +# Prints only projects that are both present in $modified_dirs and the passed +# list. +function keep-modified-projects() { + projects=${@} + for project in ${projects}; do + if echo "$modified_dirs" | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + libclc) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +# Generic pipeline for projects that have not defined custom steps. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. + +# Figure out which projects need to be built on each platform +modified_projects="$(keep-modified-projects ${all_projects})" +echo "modified_projects: $modified_projects" + +if [ "${RUNNER_OS}" = "Linux" ]; then + projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects})) +elif [ "${RUNNER_OS}" = "Windows" ]; then + projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects})) +else + echo "Unknown runner OS: $RUNNER_OS" +fi +check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq) +projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq) + +echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT +echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT + +cat $GITHUB_OUTPUT diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml new file mode 100644 index 00000000000000..4864ab72241dde --- /dev/null +++ b/.github/workflows/precommit-windows.yml @@ -0,0 +1,61 @@ +name: "Windows Precommit Tests" + +permissions: + contents: read + +on: + pull_request: + branches: + - main + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + build-llvm-windows: + name: "Build and test LLVM (Windows)" + runs-on: windows-2022 + steps: + - name: Setup Windows + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + - name: Fetch LLVM sources + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1 + with: + max-size: 500M + variant: sccache + key: precommit-windows + - name: Compute projects to test + id: compute-projects + uses: ./.github/workflows/compute-projects-to-test + with: + projects: clang llvm lld + + - name: Configure LLVM + shell: bash + if: ${{ steps.compute-projects.outputs.check-targets }} + run: | + cmake -B build -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_LIT_ARGS="-v --no-progress-bar" \ + -S llvm + - name: Build LLVM + if: ${{ steps.compute-projects.outputs.check-targets }} + run: | + ninja -C build test-depends + - name: Check LLVM + if: ${{ steps.compute-projects.outputs.check-targets }} + run: | + ninja -C build "${{ steps.compute-projects.outputs.check-targets }}" From 673c0788673e27ca43dcc869e0506adc1ec87a30 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 17 Feb 2024 05:04:31 +0000 Subject: [PATCH 02/38] Workflow improvements: * Split the build into 3 jobs to avoid timeouts. * Maintiain an scacche cache that is unique for each PR. --- .../workflows/pr-sccache-restore/action.yml | 44 +++++ .github/workflows/pr-sccache-save/action.yml | 38 ++++ .github/workflows/precommit-windows.yml | 171 ++++++++++++++++-- 3 files changed, 236 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/pr-sccache-restore/action.yml create mode 100644 .github/workflows/pr-sccache-save/action.yml diff --git a/.github/workflows/pr-sccache-restore/action.yml b/.github/workflows/pr-sccache-restore/action.yml new file mode 100644 index 00000000000000..9b620c7bdcbbe6 --- /dev/null +++ b/.github/workflows/pr-sccache-restore/action.yml @@ -0,0 +1,44 @@ +name: PR sccache restore + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + id: artifact-url + with: + script: | + const data = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'sccache-pr' + context.issue.number + }) + + console.log(data.data.artifacts) + + for (artifact of data.data.artifacts) { + if (artifact.expired) { + continue; + } + console.log(artifact) + const url = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip" + }) + console.log(url.url) + return url.url + } + console.log("Could not find previous sccache for this PR.") + return "" + + - shell: bash + if: steps.artifact-url.outputs.result != '' + run: | + curl -L -o sccache-pr${{ github.event.number }}.zip ${{ steps.artifact-url.outputs.result }} + # Is this the best way to clear the cache? + rm -Rf .sccache/ + unzip -d .sccache sccache-pr${{ github.event.number}}.zip + ls -ltr + + diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml new file mode 100644 index 00000000000000..43f0cb29115b87 --- /dev/null +++ b/.github/workflows/pr-sccache-save/action.yml @@ -0,0 +1,38 @@ +name: PR sccache save + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + with: + script: | + const data = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'sccache-pr' + context.issue.number + }) + + console.log(data.data.artifacts) + if (data.data.artifacts.length == 0) { + return ''; + } + console.log(data.data.artifacts[0]) + const artifact_id = data.data.artifacts[0].id + + // Delete the exisitng artifact so we can upload a new one with the same name. + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact_id + }) + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + with: + name: 'sccache-pr${{ github.event.number }}' + path: .sccache + retention-days: 7 + + - shell: bash + run: | + sccache --show-stats + diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 4864ab72241dde..ddcaa656215b3c 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -15,47 +15,184 @@ concurrency: cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: - build-llvm-windows: - name: "Build and test LLVM (Windows)" + compute-projects: + name: "Compute Projects to Test" + runs-on: ubuntu-22.04 + outputs: + projects: ${{ steps.vars.outputs.projects }} + check-targets: ${{ steps.vars.outputs.check-targets }} + test-build: ${{ steps.vars.outputs.check-targets != '' }} + steps: + - name: Fetch LLVM sources + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Compute projects to test + id: vars + uses: ./.github/workflows/compute-projects-to-test + + build-windows-part1: + name: "Build LLVM/Clang" runs-on: windows-2022 + needs: + - compute-projects + if: ${{ needs.compute-projects.outputs.test-build == 'true' }} steps: - name: Setup Windows uses: llvm/actions/setup-windows@main with: arch: amd64 + - name: Fetch LLVM sources uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: Setup ccache + + - name: Setup sccache uses: hendrikmuhs/ccache-action@v1 with: - max-size: 500M + max-size: 2G variant: sccache key: precommit-windows + + - name: Restore sccache from previous PR run + uses: ./.github/workflows/pr-sccache-restore + - name: Compute projects to test id: compute-projects uses: ./.github/workflows/compute-projects-to-test - with: - projects: clang llvm lld - name: Configure LLVM shell: bash - if: ${{ steps.compute-projects.outputs.check-targets }} run: | cmake -B build -GNinja \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_PROJECTS="${{ steps.compute-projects.outputs.projects }}" \ - -DCMAKE_C_COMPILER_LAUNCHER=sccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -DLLVM_ENABLE_PROJECTS="${{ needs.compute-projects.outputs.projects }}" \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_LIT_ARGS="-v --no-progress-bar" \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ -S llvm - - name: Build LLVM - if: ${{ steps.compute-projects.outputs.check-targets }} + + - name: Build + shell: bash + run: | + targets="llc" + if echo "${{ needs.compute-projects.outputs.check-targets }}" | grep clang; then + targets="$targets clang" + fi + ninja -C build $targets + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save + + - name: Package Build Directory + shell: bash run: | - ninja -C build test-depends + tar -c . | zstd -T0 -c > ../llvm-project.tar.zst + mv ../llvm-project.tar.zst . + + - name: Upload Build + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + with: + name: part1 + path: llvm-project.tar.zst + retention-days: 2 + + build-windows-part2: + name: "Build All" + needs: + - build-windows-part1 + runs-on: windows-2022 + steps: + - name: Setup Windows + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1 + with: + max-size: 2G + variant: sccache + key: precommit-windows + # The llvm-project.zip archive contains the sccache directory from + # part 1, so we don't need to restore the cache. + restore: false + + - name: Download Artifact + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + pattern: part1 + + - name: Unpack Artifact + shell: bash + run: | + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst + + - name: Build + shell: bash + run: | + ls + ninja -C build + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save + + - name: Package Build Directory + shell: bash + run: | + tar -c . | zstd -T0 -c > ../llvm-project.tar.zst + mv ../llvm-project.tar.zst . + + - name: Upload Build + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + with: + name: part2 + path: llvm-project.tar.zst + retention-days: 2 + + test-windows: + name: "Test" + needs: + - compute-projects + - build-windows-part2 + runs-on: windows-2022 + steps: + - name: Setup Windows + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1 + with: + max-size: 2G + variant: sccache + key: precommit-windows + # The llvm-project.zip archive contains the sccache directory from + # part 1, so we don't need to restore the cache. + restore: false + + - name: Download Artifact + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + pattern: part2 + merge-multiple: true + + - name: Unpack Artifact + shell: bash + run: | + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst + - name: Check LLVM - if: ${{ steps.compute-projects.outputs.check-targets }} + if: ${{ needs.compute-projects.outputs.check-targets }} run: | - ninja -C build "${{ steps.compute-projects.outputs.check-targets }}" + ninja -C build ${{ needs.compute-projects.outputs.check-targets }} + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save From 985fc7e1a7d957b045e3de1193f2b68120394b55 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 27 Apr 2024 15:02:30 -0700 Subject: [PATCH 03/38] DEBUG Add a file so that the CI triggers --- llvm/a | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 llvm/a diff --git a/llvm/a b/llvm/a new file mode 100644 index 00000000000000..e69de29bb2d1d6 From fe76c849d1e44600367eb5c8915496b3be47b143 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 06:38:36 -0700 Subject: [PATCH 04/38] Fix sccache save --- .github/workflows/precommit-windows.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index ddcaa656215b3c..2ee8c380474107 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -2,6 +2,7 @@ name: "Windows Precommit Tests" permissions: contents: read + actions: read on: pull_request: @@ -53,6 +54,10 @@ jobs: max-size: 2G variant: sccache key: precommit-windows + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save - name: Restore sccache from previous PR run uses: ./.github/workflows/pr-sccache-restore From 362959af994a1b424addfb6cb81d150838973710 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 06:46:22 -0700 Subject: [PATCH 05/38] Fix artifact saving --- .github/workflows/pr-sccache-save/action.yml | 2 +- .github/workflows/precommit-windows.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index 43f0cb29115b87..12eafbb58f47db 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -6,7 +6,7 @@ runs: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 with: script: | - const data = await github.rest.actions.listWorkflowRunArtifacts({ + const data = await github.rest.actions.listArtifactsForRepo({ owner: context.repo.owner, repo: context.repo.repo, name: 'sccache-pr' + context.issue.number diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 2ee8c380474107..e2d545acfd817d 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -2,7 +2,6 @@ name: "Windows Precommit Tests" permissions: contents: read - actions: read on: pull_request: From f3bdb5f80749105a3011e3e267b38bd3fdcf0a64 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 06:56:19 -0700 Subject: [PATCH 06/38] Revert "Fix artifact saving" This reverts commit 362959af994a1b424addfb6cb81d150838973710. --- .github/workflows/pr-sccache-save/action.yml | 2 +- .github/workflows/precommit-windows.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index 12eafbb58f47db..43f0cb29115b87 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -6,7 +6,7 @@ runs: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 with: script: | - const data = await github.rest.actions.listArtifactsForRepo({ + const data = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, name: 'sccache-pr' + context.issue.number diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index e2d545acfd817d..2ee8c380474107 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -2,6 +2,7 @@ name: "Windows Precommit Tests" permissions: contents: read + actions: read on: pull_request: From 9d2c6abf22294bb1e68a132f8c6f4f4d55515169 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 06:58:05 -0700 Subject: [PATCH 07/38] Fix artifact saving --- .github/workflows/pr-sccache-save/action.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index 43f0cb29115b87..b8959b81dbe676 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -6,11 +6,16 @@ runs: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 with: script: | - const data = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - name: 'sccache-pr' + context.issue.number - }) + try { + const data = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'sccache-pr' + context.issue.number + }) + } catch { + console.log("No existing sccache artifact"); + return ''; + } console.log(data.data.artifacts) if (data.data.artifacts.length == 0) { From ef7e8eaf0ee8986efc08b73de2f9ce69d51d9ae4 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 07:42:24 -0700 Subject: [PATCH 08/38] Fix sccache savign --- .github/workflows/pr-sccache-save/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index b8959b81dbe676..12bd656502a9bd 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -10,6 +10,7 @@ runs: const data = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, + run_id: context.runId, name: 'sccache-pr' + context.issue.number }) } catch { From 25587485fcea1eda9e3421fa80912a16dc9bc2d8 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 07:48:27 -0700 Subject: [PATCH 09/38] Fix sccache savign again --- .github/workflows/pr-sccache-save/action.yml | 27 ++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index 12bd656502a9bd..74da5e92f7e9ee 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -13,24 +13,25 @@ runs: run_id: context.runId, name: 'sccache-pr' + context.issue.number }) + + console.log(data.data.artifacts) + if (data.data.artifacts.length == 0) { + return ''; + } + console.log(data.data.artifacts[0]) + const artifact_id = data.data.artifacts[0].id + + // Delete the exisitng artifact so we can upload a new one with the same name. + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact_id + }) } catch { console.log("No existing sccache artifact"); return ''; } - console.log(data.data.artifacts) - if (data.data.artifacts.length == 0) { - return ''; - } - console.log(data.data.artifacts[0]) - const artifact_id = data.data.artifacts[0].id - - // Delete the exisitng artifact so we can upload a new one with the same name. - github.rest.actions.deleteArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact_id - }) - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 with: From 0a5b0b52c524f4b3b624f093d2fdee45bba5a300 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 08:11:52 -0700 Subject: [PATCH 10/38] Fix permissions --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 2ee8c380474107..a4ec1bf6665b0e 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -2,7 +2,7 @@ name: "Windows Precommit Tests" permissions: contents: read - actions: read + actions: write on: pull_request: From ad52820299536eb2da51721120974ebe558d99a0 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 08:12:27 -0700 Subject: [PATCH 11/38] Fix delte --- .github/workflows/pr-sccache-save/action.yml | 39 +++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml index 74da5e92f7e9ee..def50fbc14d537 100644 --- a/.github/workflows/pr-sccache-save/action.yml +++ b/.github/workflows/pr-sccache-save/action.yml @@ -6,31 +6,26 @@ runs: - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 with: script: | - try { - const data = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - name: 'sccache-pr' + context.issue.number - }) + const data = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + name: 'sccache-pr' + context.issue.number + }) - console.log(data.data.artifacts) - if (data.data.artifacts.length == 0) { - return ''; - } - console.log(data.data.artifacts[0]) - const artifact_id = data.data.artifacts[0].id - - // Delete the exisitng artifact so we can upload a new one with the same name. - github.rest.actions.deleteArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact_id - }) - } catch { - console.log("No existing sccache artifact"); + console.log(data.data.artifacts) + if (data.data.artifacts.length == 0) { return ''; } + console.log(data.data.artifacts[0]) + const artifact_id = data.data.artifacts[0].id + + // Delete the exisiting artifact so we can upload a new one with the same name. + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact_id + }) - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 From 2a3d96f5dd6258d2697f348856047106b8cefd73 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 09:21:31 -0700 Subject: [PATCH 12/38] XXX: debug --- .github/workflows/precommit-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index a4ec1bf6665b0e..243eeac2738c05 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -94,6 +94,7 @@ jobs: - name: Package Build Directory shell: bash run: | + ls tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . From 4aeeb91d1d94e551c311fa287a44dfc654b43e4c Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 09:45:18 -0700 Subject: [PATCH 13/38] XXX:Debug --- .github/workflows/precommit-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 243eeac2738c05..12561338e99ee9 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -134,6 +134,7 @@ jobs: - name: Unpack Artifact shell: bash run: | + ls tar --zstd -xf llvm-project.tar.zst rm llvm-project.tar.zst From 57d8647e6fb31d6f09d24d9b3b8458e0ab185a2e Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 10:16:07 -0700 Subject: [PATCH 14/38] Fix directory --- .github/workflows/precommit-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 12561338e99ee9..a88106237ea15a 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -130,6 +130,7 @@ jobs: uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 with: pattern: part1 + merge-multiple: true - name: Unpack Artifact shell: bash From f743da6993c109845885ca86c849c865406065e4 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 14:19:34 -0700 Subject: [PATCH 15/38] Port to auto-retry --- .github/workflows/precommit-windows.yml | 112 ++++++++++++------------ 1 file changed, 54 insertions(+), 58 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index a88106237ea15a..7b9432519f01ef 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -34,12 +34,22 @@ jobs: uses: ./.github/workflows/compute-projects-to-test build-windows-part1: - name: "Build LLVM/Clang" + name: "Build" runs-on: windows-2022 + outputs: + job-id: ${{ steps.vars.outputs.job-id }} + build-timeout: ${{ steps.build.outputs.timeout }} + build-failed: ${{ steps.build.outputs.failed }} needs: - compute-projects if: ${{ needs.compute-projects.outputs.test-build == 'true' }} steps: + - name: Collect Job ID + id: vars + shell: bash + run: | + echo job-id=${{ github.job }} >> $GITHUB_OUTPUT + - name: Setup Windows uses: llvm/actions/setup-windows@main with: @@ -55,10 +65,6 @@ jobs: variant: sccache key: precommit-windows - - name: Save sccache for next PR run - if: always() - uses: ./.github/workflows/pr-sccache-save - - name: Restore sccache from previous PR run uses: ./.github/workflows/pr-sccache-restore @@ -79,13 +85,12 @@ jobs: -S llvm - name: Build + id: build shell: bash + timeout-minutes: 1 run: | - targets="llc" - if echo "${{ needs.compute-projects.outputs.check-targets }}" | grep clang; then - targets="$targets clang" - fi - ninja -C build $targets + ninja -C build ${{ needs.compute-projects.outputs.check-targets }} || echo "failed=true" >> $GITHUB_OUTPUT + echo "timeout=false" >> $GITHUB_OUTPUT - name: Save sccache for next PR run if: always() @@ -93,23 +98,58 @@ jobs: - name: Package Build Directory shell: bash + if: always() && steps.build.outputs.timeout != 'false' run: | - ls tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . - name: Upload Build uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() && steps.build.outputs.timeout != 'false' with: name: part1 path: llvm-project.tar.zst retention-days: 2 - build-windows-part2: - name: "Build All" + check-part1-result: + name: "Check Part1 Result" + if: always() + outputs: + result: ${{ steps.part1-status.outputs.result }} needs: + - compute-projects - build-windows-part1 runs-on: windows-2022 + steps: + - name: Check Status of Part1 Build + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + id: part1-status + with: + script: | + const data = await github.rest.actions.getJobForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + job_id: "${{ needs.build-windows-part1.outputs.job-id }}" + }); + console.log(data) + if (data.data.conclusion == "success") { + return "success"; + } + + if ("${{ needs.build-windows-part1.outputs.failed }}" == "true" && + "${{ needs.build-windows-part1.outputs.timeout }}" != "false") { + // Failed due to a timeout + return "timeout"; + } + return "failure" + + build-windows: + if: ${{ needs.check-part1-result.outputs.result == 'timeout' }} + needs: + - compute-projects + - build-windows-part1 + - check-part1-result + runs-on: windows-2022 steps: - name: Setup Windows uses: llvm/actions/setup-windows@main @@ -142,8 +182,7 @@ jobs: - name: Build shell: bash run: | - ls - ninja -C build + ninja -C build ${{ needs.compute-projects.outputs.check-targets }} - name: Save sccache for next PR run if: always() @@ -161,46 +200,3 @@ jobs: name: part2 path: llvm-project.tar.zst retention-days: 2 - - test-windows: - name: "Test" - needs: - - compute-projects - - build-windows-part2 - runs-on: windows-2022 - steps: - - name: Setup Windows - uses: llvm/actions/setup-windows@main - with: - arch: amd64 - - - name: Setup ccache - uses: hendrikmuhs/ccache-action@v1 - with: - max-size: 2G - variant: sccache - key: precommit-windows - # The llvm-project.zip archive contains the sccache directory from - # part 1, so we don't need to restore the cache. - restore: false - - - name: Download Artifact - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 - with: - pattern: part2 - merge-multiple: true - - - name: Unpack Artifact - shell: bash - run: | - tar --zstd -xf llvm-project.tar.zst - rm llvm-project.tar.zst - - - name: Check LLVM - if: ${{ needs.compute-projects.outputs.check-targets }} - run: | - ninja -C build ${{ needs.compute-projects.outputs.check-targets }} - - - name: Save sccache for next PR run - if: always() - uses: ./.github/workflows/pr-sccache-save From 062585b00326b0fbfb939a1c58f9c2f680030508 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 15:43:39 -0700 Subject: [PATCH 16/38] Fixes for retry --- .github/workflows/precommit-windows.yml | 31 +++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 7b9432519f01ef..21f05a6b0cd806 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -68,11 +68,7 @@ jobs: - name: Restore sccache from previous PR run uses: ./.github/workflows/pr-sccache-restore - - name: Compute projects to test - id: compute-projects - uses: ./.github/workflows/compute-projects-to-test - - - name: Configure LLVM + - name: Configure shell: bash run: | cmake -B build -GNinja \ @@ -112,7 +108,7 @@ jobs: retention-days: 2 check-part1-result: - name: "Check Part1 Result" + name: "Check Build Result" if: always() outputs: result: ${{ steps.part1-status.outputs.result }} @@ -126,20 +122,25 @@ jobs: id: part1-status with: script: | - const data = await github.rest.actions.getJobForWorkflowRun({ + const data = await github.rest.actions.getJobsForWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, - job_id: "${{ needs.build-windows-part1.outputs.job-id }}" + run_id: "${{ github.run_id }}" }); console.log(data) - if (data.data.conclusion == "success") { - return "success"; - } + for (job of data.data.jobs) { + if (job.name != "build-windows-part1") { + continue; + } + if (job.conclusion == "success") { + return "success"; + } - if ("${{ needs.build-windows-part1.outputs.failed }}" == "true" && - "${{ needs.build-windows-part1.outputs.timeout }}" != "false") { - // Failed due to a timeout - return "timeout"; + if ("${{ needs.build-windows-part1.outputs.failed }}" == "true" && + "${{ needs.build-windows-part1.outputs.timeout }}" != "false") { + // Failed due to a timeout + return "timeout"; + } } return "failure" From 449e871b3bb5193b294922bad73c798989879813 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 15:54:37 -0700 Subject: [PATCH 17/38] xxx; fix --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 21f05a6b0cd806..75152e9f40222d 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -122,7 +122,7 @@ jobs: id: part1-status with: script: | - const data = await github.rest.actions.getJobsForWorkflowRun({ + const data = await github.rest.actions.listJobsForWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, run_id: "${{ github.run_id }}" From cdfd43d17fc5ee1faf0d4bea2d7c56d66016f7ac Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 16:03:43 -0700 Subject: [PATCH 18/38] Fix --- .github/workflows/precommit-windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 75152e9f40222d..31d2f7283e8a39 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -136,8 +136,8 @@ jobs: return "success"; } - if ("${{ needs.build-windows-part1.outputs.failed }}" == "true" && - "${{ needs.build-windows-part1.outputs.timeout }}" != "false") { + if ("${{ needs.build-windows-part1.outputs.build-failed }}" == "true" && + "${{ needs.build-windows-part1.outputs.build-timeout }}" != "false") { // Failed due to a timeout return "timeout"; } @@ -145,6 +145,7 @@ jobs: return "failure" build-windows: + name: "Build (Rebuild due to timeout)" if: ${{ needs.check-part1-result.outputs.result == 'timeout' }} needs: - compute-projects From eaef25ba6093d6f9525a28530783d5d0814108bc Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Mon, 29 Apr 2024 16:37:09 -0700 Subject: [PATCH 19/38] Fix typo --- .github/workflows/precommit-windows.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 31d2f7283e8a39..54718f2c1e7878 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -108,7 +108,7 @@ jobs: retention-days: 2 check-part1-result: - name: "Check Build Result" + name: "Check For Timeout" if: always() outputs: result: ${{ steps.part1-status.outputs.result }} @@ -136,7 +136,7 @@ jobs: return "success"; } - if ("${{ needs.build-windows-part1.outputs.build-failed }}" == "true" && + if ("${{ needs.build-windows-part1.outputs.build-failed }}" != "true" && "${{ needs.build-windows-part1.outputs.build-timeout }}" != "false") { // Failed due to a timeout return "timeout"; @@ -145,7 +145,7 @@ jobs: return "failure" build-windows: - name: "Build (Rebuild due to timeout)" + name: "Continue Build (due to timeout)" if: ${{ needs.check-part1-result.outputs.result == 'timeout' }} needs: - compute-projects From 5f16f98a65a64719c946fa682b9cc22faf50caa9 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:06:43 -0700 Subject: [PATCH 20/38] XXX: fixes --- .github/workflows/precommit-windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 54718f2c1e7878..adfc639c2d4025 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -129,7 +129,8 @@ jobs: }); console.log(data) for (job of data.data.jobs) { - if (job.name != "build-windows-part1") { + console.log(job) + if (job.name != "Build") { continue; } if (job.conclusion == "success") { From fc11d90e378b7f3aa0f8327ef5bbdec87dc42544 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:25:53 -0700 Subject: [PATCH 21/38] Better retry --- .github/workflows/precommit-windows.yml | 87 ++++++------------------- 1 file changed, 21 insertions(+), 66 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index adfc639c2d4025..73fa2a4528ac1e 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -44,11 +44,19 @@ jobs: - compute-projects if: ${{ needs.compute-projects.outputs.test-build == 'true' }} steps: - - name: Collect Job ID - id: vars + - name: Download Artifact + id: timeout-restore + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + pattern: timeout-save + merge-multiple: true + + - name: Unpack Artifact shell: bash run: | - echo job-id=${{ github.job }} >> $GITHUB_OUTPUT + ls + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst - name: Setup Windows uses: llvm/actions/setup-windows@main @@ -57,6 +65,7 @@ jobs: - name: Fetch LLVM sources uses: actions/checkout@v4 + if: ${{ steps.timeout-restore.outputs.download-path == '' }} - name: Setup sccache uses: hendrikmuhs/ccache-action@v1 @@ -103,11 +112,11 @@ jobs: uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 if: always() && steps.build.outputs.timeout != 'false' with: - name: part1 + name: timeout-save path: llvm-project.tar.zst retention-days: 2 - check-part1-result: + check-for-timeout: name: "Check For Timeout" if: always() outputs: @@ -115,6 +124,8 @@ jobs: needs: - compute-projects - build-windows-part1 + permissions: + actions: write runs-on: windows-2022 steps: - name: Check Status of Part1 Build @@ -139,67 +150,11 @@ jobs: if ("${{ needs.build-windows-part1.outputs.build-failed }}" != "true" && "${{ needs.build-windows-part1.outputs.build-timeout }}" != "false") { - // Failed due to a timeout - return "timeout"; + github.rest.actions.reRunJobForWorkflow({ + owner: context.repo.owner, + repo: context.repo.repo, + job_id: job.id + }) } } return "failure" - - build-windows: - name: "Continue Build (due to timeout)" - if: ${{ needs.check-part1-result.outputs.result == 'timeout' }} - needs: - - compute-projects - - build-windows-part1 - - check-part1-result - runs-on: windows-2022 - steps: - - name: Setup Windows - uses: llvm/actions/setup-windows@main - with: - arch: amd64 - - - name: Setup ccache - uses: hendrikmuhs/ccache-action@v1 - with: - max-size: 2G - variant: sccache - key: precommit-windows - # The llvm-project.zip archive contains the sccache directory from - # part 1, so we don't need to restore the cache. - restore: false - - - name: Download Artifact - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 - with: - pattern: part1 - merge-multiple: true - - - name: Unpack Artifact - shell: bash - run: | - ls - tar --zstd -xf llvm-project.tar.zst - rm llvm-project.tar.zst - - - name: Build - shell: bash - run: | - ninja -C build ${{ needs.compute-projects.outputs.check-targets }} - - - name: Save sccache for next PR run - if: always() - uses: ./.github/workflows/pr-sccache-save - - - name: Package Build Directory - shell: bash - run: | - tar -c . | zstd -T0 -c > ../llvm-project.tar.zst - mv ../llvm-project.tar.zst . - - - name: Upload Build - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - with: - name: part2 - path: llvm-project.tar.zst - retention-days: 2 From f23d25abe6a0ae1ecbc0677af05a7a8e3a959ba5 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:29:56 -0700 Subject: [PATCH 22/38] Fix bug --- .github/workflows/precommit-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 73fa2a4528ac1e..579470dc923555 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -52,6 +52,7 @@ jobs: merge-multiple: true - name: Unpack Artifact + if: ${{ steps.timeout-restore.outputs.download-path == '' }} shell: bash run: | ls From 929c425047e6d018d05c96b4051f115d07e15777 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:44:51 -0700 Subject: [PATCH 23/38] Fix --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 579470dc923555..2e7daaa724164f 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -52,7 +52,7 @@ jobs: merge-multiple: true - name: Unpack Artifact - if: ${{ steps.timeout-restore.outputs.download-path == '' }} + if: ${{ steps.timeout-restore.outputs.download-path != '' }} shell: bash run: | ls From d84c7cd58c3bfcf87c006d1e1e61fdfb6752c520 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:48:34 -0700 Subject: [PATCH 24/38] XXX: debug --- .github/workflows/precommit-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 2e7daaa724164f..687b48927d9aed 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -55,6 +55,7 @@ jobs: if: ${{ steps.timeout-restore.outputs.download-path != '' }} shell: bash run: | + echo "DOWNLOAD PATH: ${{ steps.timeout-restore.outputs.download-path }}" ls tar --zstd -xf llvm-project.tar.zst rm llvm-project.tar.zst From 644a9b995dca3506aee30a9f2685e202fd2b92ed Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:57:01 -0700 Subject: [PATCH 25/38] XXX: fix --- .github/workflows/precommit-windows.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 687b48927d9aed..25ce808ba0ff4f 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -45,20 +45,22 @@ jobs: if: ${{ needs.compute-projects.outputs.test-build == 'true' }} steps: - name: Download Artifact - id: timeout-restore uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 with: pattern: timeout-save merge-multiple: true - name: Unpack Artifact - if: ${{ steps.timeout-restore.outputs.download-path != '' }} + id: timeout-artifact shell: bash run: | - echo "DOWNLOAD PATH: ${{ steps.timeout-restore.outputs.download-path }}" - ls - tar --zstd -xf llvm-project.tar.zst - rm llvm-project.tar.zst + if [ -e llvm-project.tar.zst ]; + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi - name: Setup Windows uses: llvm/actions/setup-windows@main @@ -67,7 +69,7 @@ jobs: - name: Fetch LLVM sources uses: actions/checkout@v4 - if: ${{ steps.timeout-restore.outputs.download-path == '' }} + if: ${{ steps.timeout-artifact.outputs.exists != 'true' }} - name: Setup sccache uses: hendrikmuhs/ccache-action@v1 From 7c97e889f093fbdf251b9e01c682d10658b9e110 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 06:59:40 -0700 Subject: [PATCH 26/38] XXX: Fix --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 25ce808ba0ff4f..568b001dc2e7f0 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -54,7 +54,7 @@ jobs: id: timeout-artifact shell: bash run: | - if [ -e llvm-project.tar.zst ]; + if [ -e llvm-project.tar.zst ] then; tar --zstd -xf llvm-project.tar.zst rm llvm-project.tar.zst echo "exists=true" >> $GITHUB_OUTPUT From f9b1ffe1a02460d8ab629992606f9cb3713ac91f Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 07:02:20 -0700 Subject: [PATCH 27/38] XXX: fix --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 568b001dc2e7f0..5ada836903e9ca 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -54,7 +54,7 @@ jobs: id: timeout-artifact shell: bash run: | - if [ -e llvm-project.tar.zst ] then; + if [ -e llvm-project.tar.zst ]; then tar --zstd -xf llvm-project.tar.zst rm llvm-project.tar.zst echo "exists=true" >> $GITHUB_OUTPUT From 65b3bc2eca4ac6d7386c3b1bfc4f82beab858777 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 07:09:23 -0700 Subject: [PATCH 28/38] Renames --- .github/workflows/precommit-windows.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 5ada836903e9ca..b1e5e7e9e29c7a 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -33,7 +33,7 @@ jobs: id: vars uses: ./.github/workflows/compute-projects-to-test - build-windows-part1: + build-windows: name: "Build" runs-on: windows-2022 outputs: @@ -127,7 +127,7 @@ jobs: result: ${{ steps.part1-status.outputs.result }} needs: - compute-projects - - build-windows-part1 + - build-windows permissions: actions: write runs-on: windows-2022 @@ -152,8 +152,8 @@ jobs: return "success"; } - if ("${{ needs.build-windows-part1.outputs.build-failed }}" != "true" && - "${{ needs.build-windows-part1.outputs.build-timeout }}" != "false") { + if ("${{ needs.build-windows.outputs.build-failed }}" != "true" && + "${{ needs.build-windows.outputs.build-timeout }}" != "false") { github.rest.actions.reRunJobForWorkflow({ owner: context.repo.owner, repo: context.repo.repo, From 933c687e728ec81383a21dac92324b1e999cd248 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 08:08:36 -0700 Subject: [PATCH 29/38] Fix function --- .github/workflows/precommit-windows.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index b1e5e7e9e29c7a..0904ca3f84ec60 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -154,11 +154,14 @@ jobs: if ("${{ needs.build-windows.outputs.build-failed }}" != "true" && "${{ needs.build-windows.outputs.build-timeout }}" != "false") { - github.rest.actions.reRunJobForWorkflow({ + // This function does not exist even though it is in the document + //github.rest.actions.reRunJobForWorkflow({ + await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', { owner: context.repo.owner, repo: context.repo.repo, job_id: job.id }) + return "timeout" } } return "failure" From 39a1dfb6539a36d8a4003aa6e33c178abb4f530b Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 08:33:51 -0700 Subject: [PATCH 30/38] Move restart to another workflow --- .github/workflows/precommit-windows.yml | 65 ++++++------------------- 1 file changed, 14 insertions(+), 51 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 0904ca3f84ec60..7888c52ba47de9 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -94,12 +94,21 @@ jobs: -S llvm - name: Build - id: build shell: bash timeout-minutes: 1 run: | - ninja -C build ${{ needs.compute-projects.outputs.check-targets }} || echo "failed=true" >> $GITHUB_OUTPUT - echo "timeout=false" >> $GITHUB_OUTPUT + touch timeout + ninja -C build ${{ needs.compute-projects.outputs.check-targets }} + rm timeout + + - name: Upload Timeout Message + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + id: timeout + if: always() + with: + name: timeout + path: timeout + retention-days: 2 - name: Save sccache for next PR run if: always() @@ -107,61 +116,15 @@ jobs: - name: Package Build Directory shell: bash - if: always() && steps.build.outputs.timeout != 'false' + if: always() && steps.timeoutoutputs.artifact-id != '' run: | tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . - name: Upload Build uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - if: always() && steps.build.outputs.timeout != 'false' + if: always() && steps.timeoutoutputs.artifact-id != '' with: name: timeout-save path: llvm-project.tar.zst retention-days: 2 - - check-for-timeout: - name: "Check For Timeout" - if: always() - outputs: - result: ${{ steps.part1-status.outputs.result }} - needs: - - compute-projects - - build-windows - permissions: - actions: write - runs-on: windows-2022 - steps: - - name: Check Status of Part1 Build - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 - id: part1-status - with: - script: | - const data = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: "${{ github.run_id }}" - }); - console.log(data) - for (job of data.data.jobs) { - console.log(job) - if (job.name != "Build") { - continue; - } - if (job.conclusion == "success") { - return "success"; - } - - if ("${{ needs.build-windows.outputs.build-failed }}" != "true" && - "${{ needs.build-windows.outputs.build-timeout }}" != "false") { - // This function does not exist even though it is in the document - //github.rest.actions.reRunJobForWorkflow({ - await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', { - owner: context.repo.owner, - repo: context.repo.repo, - job_id: job.id - }) - return "timeout" - } - } - return "failure" From 8cd89ca8ead5bbc6d1cf4ac3464d96738ffda2b2 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 08:35:20 -0700 Subject: [PATCH 31/38] Fix --- .github/workflows/precommit-windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 7888c52ba47de9..9fc5031a0f1300 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -116,14 +116,15 @@ jobs: - name: Package Build Directory shell: bash - if: always() && steps.timeoutoutputs.artifact-id != '' + if: always() && steps.timeout.outputs.artifact-id != '' run: | + echo "TMEOUT: ${{ steps.timeout.outputs.artifact-id }}" tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . - name: Upload Build uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - if: always() && steps.timeoutoutputs.artifact-id != '' + if: always() && steps.timeout.outputs.artifact-id != '' with: name: timeout-save path: llvm-project.tar.zst From 64d5cfb844d18bd7ac6856c759253dc32aaac25e Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 09:50:16 -0700 Subject: [PATCH 32/38] Fixes --- .github/workflows/precommit-windows.yml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 9fc5031a0f1300..53984e7017221d 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -37,7 +37,6 @@ jobs: name: "Build" runs-on: windows-2022 outputs: - job-id: ${{ steps.vars.outputs.job-id }} build-timeout: ${{ steps.build.outputs.timeout }} build-failed: ${{ steps.build.outputs.failed }} needs: @@ -47,7 +46,7 @@ jobs: - name: Download Artifact uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 with: - pattern: timeout-save + pattern: timeout merge-multiple: true - name: Unpack Artifact @@ -95,20 +94,11 @@ jobs: - name: Build shell: bash - timeout-minutes: 1 + id: build + timeout-minutes: 330 run: | - touch timeout ninja -C build ${{ needs.compute-projects.outputs.check-targets }} - rm timeout - - - name: Upload Timeout Message - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - id: timeout - if: always() - with: - name: timeout - path: timeout - retention-days: 2 + echo "completed=true" >> $GITHUB_OUTPUT - name: Save sccache for next PR run if: always() @@ -116,7 +106,7 @@ jobs: - name: Package Build Directory shell: bash - if: always() && steps.timeout.outputs.artifact-id != '' + if: always() && steps.build.outputs.completed != 'true' run: | echo "TMEOUT: ${{ steps.timeout.outputs.artifact-id }}" tar -c . | zstd -T0 -c > ../llvm-project.tar.zst @@ -124,8 +114,8 @@ jobs: - name: Upload Build uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - if: always() && steps.timeout.outputs.artifact-id != '' + if: always() && steps.build.outputs.completed != 'true' with: - name: timeout-save + name: timeout path: llvm-project.tar.zst retention-days: 2 From 8b2753df5b581eb8783ab635af4c4c9b86d1b776 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 15:31:01 -0700 Subject: [PATCH 33/38] XXX: Fix --- .github/workflows/precommit-windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 53984e7017221d..27963df06c36f5 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -97,8 +97,9 @@ jobs: id: build timeout-minutes: 330 run: | - ninja -C build ${{ needs.compute-projects.outputs.check-targets }} + status=`ninja -C build ${{ needs.compute-projects.outputs.check-targets }}` echo "completed=true" >> $GITHUB_OUTPUT + exit $status - name: Save sccache for next PR run if: always() @@ -108,7 +109,6 @@ jobs: shell: bash if: always() && steps.build.outputs.completed != 'true' run: | - echo "TMEOUT: ${{ steps.timeout.outputs.artifact-id }}" tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . From 2e8c48bac657e3909dd564379ee42a177a2da488 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 15:59:46 -0700 Subject: [PATCH 34/38] Fix --- .github/workflows/precommit-windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 27963df06c36f5..cbc6c6ae489a04 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -97,9 +97,9 @@ jobs: id: build timeout-minutes: 330 run: | - status=`ninja -C build ${{ needs.compute-projects.outputs.check-targets }}` + ninja -C build ${{ needs.compute-projects.outputs.check-targets }} && pass=1 echo "completed=true" >> $GITHUB_OUTPUT - exit $status + [ $pass ] || false - name: Save sccache for next PR run if: always() From be0f0404684cbba6d756c45e003f905ae386c755 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 20:03:03 -0700 Subject: [PATCH 35/38] Keep alive 9 --- .github/workflows/precommit-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index cbc6c6ae489a04..f461db3ac2ea47 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -97,7 +97,7 @@ jobs: id: build timeout-minutes: 330 run: | - ninja -C build ${{ needs.compute-projects.outputs.check-targets }} && pass=1 + ninja -C build -k 0 ${{ needs.compute-projects.outputs.check-targets }} && pass=1 echo "completed=true" >> $GITHUB_OUTPUT [ $pass ] || false From dcaaeeec6bd150a7637a451132915cc0cd8c3463 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 20:16:00 -0700 Subject: [PATCH 36/38] Fix retry --- .github/workflows/precommit-windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index f461db3ac2ea47..137dd23f4b0ab3 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -97,6 +97,7 @@ jobs: id: build timeout-minutes: 330 run: | + echo "completed=false" >> $GITHUB_OUTPUT ninja -C build -k 0 ${{ needs.compute-projects.outputs.check-targets }} && pass=1 echo "completed=true" >> $GITHUB_OUTPUT [ $pass ] || false @@ -107,14 +108,14 @@ jobs: - name: Package Build Directory shell: bash - if: always() && steps.build.outputs.completed != 'true' + if: always() && steps.build.outputs.completed == 'false' run: | tar -c . | zstd -T0 -c > ../llvm-project.tar.zst mv ../llvm-project.tar.zst . - name: Upload Build uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - if: always() && steps.build.outputs.completed != 'true' + if: always() && steps.build.outputs.completed == 'false' with: name: timeout path: llvm-project.tar.zst From 4a3bf765b832bd810bcc83efdcdeabbc306c8d9b Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Tue, 30 Apr 2024 20:27:32 -0700 Subject: [PATCH 37/38] Fix return --- .github/workflows/pr-sccache-restore/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-sccache-restore/action.yml b/.github/workflows/pr-sccache-restore/action.yml index 9b620c7bdcbbe6..abae90f7929c84 100644 --- a/.github/workflows/pr-sccache-restore/action.yml +++ b/.github/workflows/pr-sccache-restore/action.yml @@ -30,7 +30,6 @@ runs: return url.url } console.log("Could not find previous sccache for this PR.") - return "" - shell: bash if: steps.artifact-url.outputs.result != '' From 268d95f8741b20d4bf0b3e35e461c10c38f10650 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 1 May 2024 07:32:37 -0700 Subject: [PATCH 38/38] Update permissions --- .github/workflows/precommit-windows.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml index 137dd23f4b0ab3..7d06a5853f47ab 100644 --- a/.github/workflows/precommit-windows.yml +++ b/.github/workflows/precommit-windows.yml @@ -36,6 +36,8 @@ jobs: build-windows: name: "Build" runs-on: windows-2022 + permissions: + actions: write #pr-sccache-save may delete artifacts. outputs: build-timeout: ${{ steps.build.outputs.timeout }} build-failed: ${{ steps.build.outputs.failed }}