Cache trivy (#2910)

* Cache trivy Fixes #2904 Signed-off-by: Dimitris Karakasilis <[email protected]> * Cache trivy in more pipelines Signed-off-by: Dimitris Karakasilis <[email protected]> * Populate trivy cache Signed-off-by: Dimitris Karakasilis <[email protected]> * Create the trivy cache dir if it wasn't created because if there is no cache to restore, the directory doesn't get created Signed-off-by: Dimitris Karakasilis <[email protected]> * Try to gate all jobs with trivy cache job to avoid failing later. Also, since jobs run in parallel, they may start populating the cache at the same time which will result in multiple requests for the trivy database, making the caching mechanism useless. Doing it once before everything should solve this. Signed-off-by: Dimitris Karakasilis <[email protected]> --------- Signed-off-by: Dimitris Karakasilis <[email protected]>
kairos-io · Oct 3, 2024 · 7cf2177 · 7cf2177
1 parent f55def8
commit 7cf2177
Show file tree

Hide file tree

Showing 8 changed files with 188 additions and 12 deletions.
diff --git a/.github/workflows/image-pr.yaml b/.github/workflows/image-pr.yaml
@@ -12,9 +12,31 @@ env:
   FORCE_COLOR: 1
   EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
 jobs:
+  # Populate the trivy cache once for all later jobs to use
+  trivy-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
+      - name: Install earthly
+        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
+        with:
+          repository: quay.io/kairos/packages
+          packages: utils/earthly
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
   core-ubuntu-22-lts:
     uses: ./.github/workflows/reusable-build-flavor.yaml
     secrets: inherit
+    needs:
+      - trivy-cache
     permissions:
       contents: write
       security-events: write
@@ -42,6 +64,8 @@ jobs:
   core-ubuntu-24-lts:
     uses: ./.github/workflows/reusable-build-flavor.yaml
     secrets: inherit
+    needs:
+      - trivy-cache
     permissions:
       contents: write
       security-events: write
@@ -69,6 +93,8 @@ jobs:
   core-alpine:
     uses: ./.github/workflows/reusable-build-flavor.yaml
     secrets: inherit
+    needs:
+      - trivy-cache
     permissions:
       contents: write
       security-events: write
@@ -96,6 +122,8 @@ jobs:
   standard:
     uses: ./.github/workflows/reusable-build-provider.yaml
     secrets: inherit
+    needs:
+      - trivy-cache
     permissions:
       id-token: write  # OIDC support
       contents: write
@@ -128,6 +156,7 @@ jobs:
       flavor_release: "24.04"
       secureboot: false
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   install-target:
@@ -138,6 +167,7 @@ jobs:
       flavor_release: "24.04"
       secureboot: false
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   install-secureboot:
@@ -148,6 +178,7 @@ jobs:
       flavor_release: "24.04"
       secureboot: true
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   install-alpine:
@@ -157,6 +188,7 @@ jobs:
       flavor: alpine
       flavor_release: "3.19"
     needs:
+      - trivy-cache
       - core-alpine
 
   zfs:
@@ -166,6 +198,7 @@ jobs:
       flavor: ubuntu
       flavor_release: "22.04"
     needs:
+      - trivy-cache
       - core-ubuntu-22-lts
 
   acceptance:
@@ -175,6 +208,7 @@ jobs:
       flavor: ubuntu
       flavor_release: "24.04"
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   acceptance-alpine:
@@ -184,6 +218,7 @@ jobs:
       flavor: alpine
       flavor_release: "3.19"
     needs:
+      - trivy-cache
       - core-alpine
 
   bundles:
@@ -193,6 +228,7 @@ jobs:
       flavor: ubuntu
       flavor_release: "24.04"
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   reset:
@@ -202,6 +238,7 @@ jobs:
       flavor: ubuntu
       flavor_release: "24.04"
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   reset-alpine:
@@ -211,6 +248,7 @@ jobs:
       flavor: alpine
       flavor_release: "3.19"
     needs:
+      - trivy-cache
       - core-alpine
 
   netboot:
@@ -224,6 +262,7 @@ jobs:
       model: generic
       variant: core
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   netboot-alpine:
@@ -237,6 +276,7 @@ jobs:
       model: generic
       variant: core
     needs:
+      - trivy-cache
       - core-alpine
 
   upgrade:
@@ -246,6 +286,7 @@ jobs:
       flavor: ubuntu
       flavor_release: "24.04"
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   upgrade-alpine:
@@ -255,6 +296,7 @@ jobs:
       flavor: alpine
       flavor_release: "3.19"
     needs:
+      - trivy-cache
       - core-alpine
 
   upgrade-latest:
@@ -266,6 +308,7 @@ jobs:
       family: "ubuntu"
       # release_matcher: "23.10"  # introduced so tests can be green while we wait for the kairos release with the latest flavor release
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
 
   # enable once the first alpine only release is out as it currently cannot find the latest alpine release properly
@@ -275,6 +318,7 @@ jobs:
   #     flavor: alpine
   #     flavor_release: "3.19"
   #   needs:
+  #     - trivy-cache
   #     - core-alpine
 
   custom-partitioning:
@@ -299,6 +343,7 @@ jobs:
       flavor: ${{ matrix.flavor }}
       flavor_release: ${{ matrix.flavorRelease }}
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
     strategy:
       fail-fast: true
@@ -315,6 +360,7 @@ jobs:
       flavor_release: "24.04"
       label: ${{ matrix.label }}
     needs:
+      - trivy-cache
       - core-ubuntu-24-lts
     strategy:
       fail-fast: true
@@ -334,6 +380,7 @@ jobs:
       flavor_release: "3.19"
       label: ${{ matrix.label }}
     needs:
+      - trivy-cache
       - core-alpine
     strategy:
       fail-fast: true

diff --git a/.github/workflows/release-arm.yaml b/.github/workflows/release-arm.yaml
@@ -77,6 +77,27 @@ jobs:
           # end of optional handling for multi line json
           echo "::set-output name=matrix::{\"include\": $content }"
 
+  # Populate the trivy cache once for all later jobs to use
+  trivy-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
+      - name: Install earthly
+        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
+        with:
+          repository: quay.io/kairos/packages
+          packages: utils/earthly
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
+
   build-nvidia-base:
     runs-on: ARM64
     steps:
@@ -173,6 +194,7 @@ jobs:
   build-arm-core:
     runs-on: ${{ matrix.worker }}
     needs:
+      - trivy-cache
       - get-core-matrix
     permissions:
       id-token: write  # OIDC support
@@ -247,10 +269,19 @@ jobs:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
       - name: Build  🔧
         run: |
           earthly -P +all-arm \
             -VARIANT=core \
+            --TRIVY_CACHE_DIR=.trivy \
             -MODEL=${{ matrix.model }} \
             -FLAVOR=${{ matrix.flavor }} \
             -FLAVOR_RELEASE=${{ matrix.flavorRelease }} \
@@ -315,6 +346,7 @@ jobs:
   build-arm-standard:
     runs-on: ARM64
     needs:
+      - trivy-cache
       - get-standard-matrix
     permissions:
       id-token: write  # OIDC support
@@ -359,10 +391,19 @@ jobs:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_PASSWORD }}
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
       - name: Build  🔧
         run: |
           earthly -P +all-arm \
             -VARIANT=standard \
+            --TRIVY_CACHE_DIR=.trivy \
             -MODEL=${{ matrix.model }} \
             -K3S_VERSION=${{ matrix.k3s_version }} \
             -FLAVOR=${{ matrix.flavor }} \

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -93,11 +93,31 @@ jobs:
 
           # end of optional handling for multi line json
           echo "::set-output name=matrix::{\"include\": $content }"
-
+  # Populate the trivy cache once for all later jobs to use
+  trivy-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
+      - name: Install earthly
+        uses: Luet-lab/luet-install-action@cec77490c3f2416d7d07a47cfab04d448641d7ce # v1.1
+        with:
+          repository: quay.io/kairos/packages
+          packages: utils/earthly
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
   build-core:
     runs-on: ubuntu-latest
     needs:
-    - get-core-matrix
+      - trivy-cache
+      - get-core-matrix
     permissions:
       id-token: write  # OIDC support
       contents: write
@@ -161,9 +181,18 @@ jobs:
         with:
           repository: quay.io/kairos/packages
           packages: utils/earthly
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
       - name: Build  🔧
         run: |
           earthly +all  \
+            --TRIVY_CACHE_DIR=.trivy \
             --VARIANT=${{ matrix.variant }} \
             --FAMILY=${{ matrix.family }} \
             --FLAVOR=${{ matrix.flavor }} \
@@ -232,6 +261,7 @@ jobs:
       actions: read
       security-events: write
     needs:
+      - trivy-cache
       - get-uki-matrix
     strategy:
       matrix: ${{ fromJson(needs.get-uki-matrix.outputs.matrix) }}
@@ -345,7 +375,8 @@ jobs:
   build-standard:
     runs-on: ubuntu-latest
     needs:
-    - get-standard-matrix
+      - trivy-cache
+      - get-standard-matrix
     permissions:
       id-token: write  # OIDC support
       contents: write
@@ -407,9 +438,18 @@ jobs:
           packages: utils/earthly
       - name: Login to Quay Registry
         run: echo ${{ secrets.QUAY_PASSWORD }} | docker login -u ${{ secrets.QUAY_USERNAME }} --password-stdin quay.io
+      - name: Restore trivy cache
+        uses: yogeshlonkar/trivy-cache-action@v0
+        with:
+          gh-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Populate trivy Cache
+        run: |
+          [ ! -d ".trivy" ] && mkdir -p ".trivy"
+          earthly +trivy-download-db --DIR .trivy
       - name: Build  🔧
         run: |
           earthly +all  \
+            --TRIVY_CACHE_DIR=.trivy \
             --VARIANT=${{ matrix.variant }} \
             --FAMILY=${{ matrix.family }} \
             --FLAVOR=${{ matrix.flavor }} \