From 3e3cf4f34b5e5a926f086a4384e957f837245b58 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 10:44:13 -0700
Subject: [PATCH 01/16] ci: convert e2e to matrix

---
 .github/actions/e2e.yml     | 114 ++++++++++++++++++++++++++++++++++++
 .github/workflows/build.yml |  93 -----------------------------
 .github/workflows/e2e.yml   |  26 ++++++++
 3 files changed, 140 insertions(+), 93 deletions(-)
 create mode 100644 .github/actions/e2e.yml
 create mode 100644 .github/workflows/e2e.yml

diff --git a/.github/actions/e2e.yml b/.github/actions/e2e.yml
new file mode 100644
index 0000000000..fa3b4e1174
--- /dev/null
+++ b/.github/actions/e2e.yml
@@ -0,0 +1,114 @@
+name: Reusable E2E Testing Workflow
+on:
+  workflow_call:
+    inputs:
+      make_target:
+        description: 'Makefile target to execute in the Start Test step'
+        required: true
+        type: string
+      timeout_minutes:
+        description: 'The maximum number of minutes the job can run'
+        type: number
+        default: 25
+      run:
+        description: 'Whether to run the job or not'
+        required: true
+        type: boolean
+      runs_on:
+        description: 'The runner to use for the job'
+        required: true
+        type: string
+        default: 'ubuntu-20.04'
+
+jobs:
+  e2e-test:
+    if: ${{ run }}
+    runs-on: ${{ runs_on }}
+    timeout-minutes: ${{ inputs.timeout_minutes }}
+    steps:
+      - uses: actions/checkout@v4
+      
+      # configure docker to use the containerd snapshotter
+      # so that we can use the buildkit cache
+      - uses: depot/use-containerd-snapshotter-action@v1
+
+      - name: Login to Docker Hub registry
+        uses: docker/login-action@v2
+        if: (github.event_name == 'push' && github.repository == 'zeta-chain/node') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'zeta-chain/node')
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_READ_ONLY }}
+
+      - name: Login to github docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Restore go cache
+        uses: actions/cache@v4
+        id: restore-go-cache
+        with:
+          path: |
+            go-cache
+          key: cache-${{ hashFiles('go.sum') }}
+          lookup-only: ${{ github.event_name != 'push' }}
+
+      - name: Inject go cache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+        with:
+          cache-map: |
+            {
+              "go-cache": "/root/.cache/go-build"
+            }
+          skip-extraction: ${{ steps.restore-go-cache.outputs.cache-hit || github.event_name != 'push' }}
+
+      # build zetanode with cache options
+      - name: Build zetanode for cache
+        uses: docker/build-push-action@v6
+        env:
+          CACHE_FROM_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache"
+          CACHE_TO_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max"
+        with:
+          context: .
+          file: ./Dockerfile-localnet
+          push: false
+          tags: zetanode:latest
+          cache-from: ${{ env.CACHE_FROM_CONFIG }}
+          cache-to: ${{ github.event_name == 'push' && env.CACHE_TO_CONFIG || '' }}
+          target: latest-runtime
+
+      - name: Start Test
+        run: make ${{ inputs.make_target }}
+
+      # use docker logs -f rather than docker attach to make sure we get the initial logs
+      - name: Watch Test
+        run: |
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" &
+          exit $(docker wait "${container_id}")
+
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          make stop-localnet
+
+      - name: Notify Slack on Failure
+        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
+
+      - name: Stop Private Network
+        if: always()
+        run: |
+          make stop-localnet
+
+      - name: Clean Up Workspace
+        if: always()
+        shell: bash
+        run: sudo rm -rf *
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0d316ebfb3..bc075e6447 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -100,96 +100,3 @@ jobs:
         if: always()
         shell: bash
         run: rm -rf *
-
-  e2e-test:
-    runs-on: ubuntu-20.04
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@v4
-      
-      # configure docker to use the containerd snapshotter
-      # so that we can use the buildkit cache
-      - uses: depot/use-containerd-snapshotter-action@v1
-
-      - name: Login to Docker Hub registry
-        uses: docker/login-action@v2
-        if: (github.event_name == 'push' && github.repository == 'zeta-chain/node') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'zeta-chain/node')
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_READ_ONLY }}
-
-      - name: Login to github docker registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Restore go cache
-        uses: actions/cache@v4
-        id: restore-go-cache
-        with:
-          path: |
-            go-cache
-          key: cache-${{ hashFiles('go.sum') }}
-          lookup-only: ${{ github.event_name != 'push' }}
-
-      - name: Inject go cache into docker
-        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
-        with:
-          cache-map: |
-            {
-              "go-cache": "/root/.cache/go-build"
-            }
-          skip-extraction: ${{ steps.restore-go-cache.outputs.cache-hit || github.event_name != 'push' }}
-
-      # build zetanode with cache options
-      - name: Build zetanode for cache
-        uses: docker/build-push-action@v6
-        env:
-          CACHE_FROM_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache"
-          CACHE_TO_CONFIG: "type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max"
-        with:
-          context: .
-          file: ./Dockerfile-localnet
-          push: false
-          tags: zetanode:latest
-          cache-from: ${{ env.CACHE_FROM_CONFIG }}
-          cache-to: ${{ github.event_name == 'push' && env.CACHE_TO_CONFIG || '' }}
-          target: latest-runtime
-
-      - name: Start Test
-        run: make start-e2e-test
-
-      # use docker logs -f rather than docker attach to make sure we get the initial logs
-      - name: Watch Test
-        run: |
-          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
-          docker logs -f "${container_id}" &
-          exit $(docker wait "${container_id}")
-
-      - name: Full Log Dump On Failure
-        if: failure()
-        run: |
-          make stop-localnet
-
-      - name: Notify Slack on Failure
-        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
-        uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
-
-      - name: Stop Private Network
-        if: always()
-        run: |
-          make stop-localnet
-
-      - name: Clean Up Workspace
-        if: always()
-        shell: bash
-        run: sudo rm -rf *
-
-
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 0000000000..c6b78d2f2b
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,26 @@
+name: e2e
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  e2e:
+    strategy:
+      matrix:
+        include:
+          - make_target: "start-e2e-test"
+            runs_on: ubuntu-20.04
+            run: true
+          - make_target: "start-upgrade-test-light"
+            runs_on: ubuntu-20.04
+            run: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'MINIMAL_CI') }}
+    uses: ./.github/actions/e2e.yml
+    with:
+      make_target: ${{ matrix.make_target }}
+      runs_on: ${{ matrix.runs_on}}
+      run: ${{ matrix.run }}

From 6dde94b629dc2370069f24ce49497276071480c0 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 11:11:21 -0700
Subject: [PATCH 02/16] workflow not action

---
 .github/workflows/e2e.yml                          | 14 +++++++-------
 .../e2e.yml => workflows/reusable-e2e.yml}         | 12 ++++++------
 2 files changed, 13 insertions(+), 13 deletions(-)
 rename .github/{actions/e2e.yml => workflows/reusable-e2e.yml} (95%)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index c6b78d2f2b..9679d05b95 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -13,14 +13,14 @@ jobs:
     strategy:
       matrix:
         include:
-          - make_target: "start-e2e-test"
-            runs_on: ubuntu-20.04
+          - make-target: "start-e2e-test"
+            runs-on: ubuntu-20.04
             run: true
-          - make_target: "start-upgrade-test-light"
-            runs_on: ubuntu-20.04
+          - make-target: "start-upgrade-test-light"
+            runs-on: ubuntu-20.04
             run: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'MINIMAL_CI') }}
-    uses: ./.github/actions/e2e.yml
+    uses: ./.github/workflows/reusable-e2e.yml
     with:
-      make_target: ${{ matrix.make_target }}
-      runs_on: ${{ matrix.runs_on}}
+      make-target: ${{ matrix.make-target }}
+      runs-on: ${{ matrix.runs-on}}
       run: ${{ matrix.run }}
diff --git a/.github/actions/e2e.yml b/.github/workflows/reusable-e2e.yml
similarity index 95%
rename from .github/actions/e2e.yml
rename to .github/workflows/reusable-e2e.yml
index fa3b4e1174..f3aedfe0ba 100644
--- a/.github/actions/e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -2,11 +2,11 @@ name: Reusable E2E Testing Workflow
 on:
   workflow_call:
     inputs:
-      make_target:
+      make-target:
         description: 'Makefile target to execute in the Start Test step'
         required: true
         type: string
-      timeout_minutes:
+      timeout-minutes:
         description: 'The maximum number of minutes the job can run'
         type: number
         default: 25
@@ -14,7 +14,7 @@ on:
         description: 'Whether to run the job or not'
         required: true
         type: boolean
-      runs_on:
+      runs-on:
         description: 'The runner to use for the job'
         required: true
         type: string
@@ -22,9 +22,9 @@ on:
 
 jobs:
   e2e-test:
-    if: ${{ run }}
-    runs-on: ${{ runs_on }}
-    timeout-minutes: ${{ inputs.timeout_minutes }}
+    if: ${{ inputs.run }}
+    runs-on: ${{ inputs.runs-on }}
+    timeout-minutes: ${{ inputs.timeout-minutes }}
     steps:
       - uses: actions/checkout@v4
       

From 287c69a6d5eed5167777379c65f27a1ac95a89c1 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 11:15:44 -0700
Subject: [PATCH 03/16] inherit secrets

---
 .github/workflows/e2e.yml          | 6 ++++++
 .github/workflows/reusable-e2e.yml | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 9679d05b95..eeda1e9469 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -24,3 +24,9 @@ jobs:
       make-target: ${{ matrix.make-target }}
       runs-on: ${{ matrix.runs-on}}
       run: ${{ matrix.run }}
+    secrets: inherit
+  ok:
+    runs-on: ubuntu-latest
+    needs: e2e
+    steps:
+      - run: echo "e2e tests passed"
diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
index f3aedfe0ba..a7e3ef2104 100644
--- a/.github/workflows/reusable-e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -21,7 +21,7 @@ on:
         default: 'ubuntu-20.04'
 
 jobs:
-  e2e-test:
+  e2e:
     if: ${{ inputs.run }}
     runs-on: ${{ inputs.runs-on }}
     timeout-minutes: ${{ inputs.timeout-minutes }}
@@ -80,7 +80,7 @@ jobs:
           target: latest-runtime
 
       - name: Start Test
-        run: make ${{ inputs.make_target }}
+        run: make ${{ inputs.make-target }}
 
       # use docker logs -f rather than docker attach to make sure we get the initial logs
       - name: Watch Test

From 5efdc854cc9914c5730628d73f3df7dd44462de0 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 13:51:33 -0700
Subject: [PATCH 04/16] update

---
 .github/workflows/e2e.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index eeda1e9469..6ce37dd1a0 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -7,6 +7,11 @@ on:
   pull_request:
     branches:
       - "*"
+  workflow_dispatch:
+  schedule:
+    # run at 6AM UTC Daily
+    # 6AM UTC -> 11PM PT
+    - cron: "0 6 * * *"
 
 jobs:
   e2e:
@@ -19,12 +24,14 @@ jobs:
           - make-target: "start-upgrade-test-light"
             runs-on: ubuntu-20.04
             run: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'MINIMAL_CI') }}
+    name: ${{ matrix.make-target }}
     uses: ./.github/workflows/reusable-e2e.yml
     with:
       make-target: ${{ matrix.make-target }}
       runs-on: ${{ matrix.runs-on}}
       run: ${{ matrix.run }}
     secrets: inherit
+  # this allows you to set a required status check
   ok:
     runs-on: ubuntu-latest
     needs: e2e

From f2929a4caef3c6277b377383fbda4838362816d2 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 14:17:11 -0700
Subject: [PATCH 05/16] test run conditionals

---
 .github/workflows/e2e.yml | 50 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 6ce37dd1a0..5e82979170 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -7,23 +7,69 @@ on:
   pull_request:
     branches:
       - "*"
+  merge_group:
   workflow_dispatch:
   schedule:
     # run at 6AM UTC Daily
     # 6AM UTC -> 11PM PT
     - cron: "0 6 * * *"
 
+concurrency: 
+  group: e2e-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
 jobs:
+  matrix-conditionals:
+    runs-on: ubuntu-22.04
+    env:
+      GH_TOKEN: ${{ github.token }}
+    outputs:
+      UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
+      UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
+      ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }}
+    steps:
+      # use cli rather than event context to avoid race conditions (label added after push)
+      - id: matrix-conditionals
+        run: |
+          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
+            labels=$(gh pr view -R ${{github.repository}} ${{github.event.pull_request.number}} --json labels -q '.labels[].name')
+            if [[ "$labels" == *"UPGRADE_TESTS"* ]]; then
+              echo "UPGRADE_TESTS=true" >> $GITHUB_OUTPUT
+            fi
+
+            if [[ "$labels" == *"UPGRADE_LIGHT_TESTS"* ]]; then
+              echo "UPGRADE_LIGHT_TESTS=true" >> $GITHUB_OUTPUT
+            fi
+
+            if [[ "$labels" == *"ADMIN_TESTS"* ]]; then
+              echo "ADMIN_TESTS=true" >> $GITHUB_OUTPUT
+            fi
+          elif [[ ${{ github.event_name }} == 'merge_group' ]]; then
+            #TODO: add ability to opt out via MINIMAL_TESTS
+            echo "UPGRADE_LIGHT_TESTS=true" >> $GITHUB_OUTPUT
+          elif [[ ${{ github.event_name }} == 'schedule' ]]; then
+            echo "UPGRADE_TESTS=true" >> $GITHUB_OUTPUT
+            echo "UPGRADE_LIGHT_TESTS=true" >> $GITHUB_OUTPUT
+            echo "ADMIN_TESTS=true" >> $GITHUB_OUTPUT
+          fi
+
   e2e:
+    needs: matrix-conditionals
     strategy:
       matrix:
         include:
           - make-target: "start-e2e-test"
             runs-on: ubuntu-20.04
             run: true
+          - make-target: "start-upgrade-test"
+            runs-on: ubuntu-20.04
+            run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}
           - make-target: "start-upgrade-test-light"
             runs-on: ubuntu-20.04
-            run: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'MINIMAL_CI') }}
+            run: ${{ needs.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS == 'true' }}
+          - make-target: "start-e2e-admin-test"
+            runs-on: ubuntu-20.04
+            run: ${{ needs.matrix-conditionals.outputs.ADMIN_TESTS == 'true' }}
     name: ${{ matrix.make-target }}
     uses: ./.github/workflows/reusable-e2e.yml
     with:
@@ -33,7 +79,7 @@ jobs:
     secrets: inherit
   # this allows you to set a required status check
   ok:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: e2e
     steps:
       - run: echo "e2e tests passed"

From d04b2d263a8bace42ac224ee7fd1e4bad8ec5251 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 14:50:33 -0700
Subject: [PATCH 06/16] fix cache

---
 .github/workflows/reusable-e2e.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
index a7e3ef2104..a66f72d10a 100644
--- a/.github/workflows/reusable-e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -53,7 +53,6 @@ jobs:
           path: |
             go-cache
           key: cache-${{ hashFiles('go.sum') }}
-          lookup-only: ${{ github.event_name != 'push' }}
 
       - name: Inject go cache into docker
         uses: reproducible-containers/buildkit-cache-dance@v3.1.2

From fe756eb12f112dcc81e791684ce30a779e2fe73e Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Tue, 2 Jul 2024 16:02:11 -0700
Subject: [PATCH 07/16] adjust ok logic (skipped does not work with required
 status checks)

---
 .github/workflows/e2e.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 5e82979170..361b52a96f 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -81,5 +81,12 @@ jobs:
   ok:
     runs-on: ubuntu-22.04
     needs: e2e
+    if: always()
     steps:
-      - run: echo "e2e tests passed"
+      - run: |
+          result="${{ needs.e2e.result }}"
+          if [[ $result == "success" || $result == "skipped" ]]; then
+            exit 0
+          else
+            exit 1
+          fi

From 07f534a3850bc0c4fa0c9c3985a7790277edcb27 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 10:45:03 -0700
Subject: [PATCH 08/16] all profile and .env for defaults

---
 .github/workflows/e2e.yml                     |   3 -
 .github/workflows/execute_advanced_tests.yaml | 106 ------------------
 Makefile                                      |   6 +-
 contrib/localnet/docker-compose.yml           |   7 ++
 4 files changed, 8 insertions(+), 114 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 361b52a96f..af85975403 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -44,9 +44,6 @@ jobs:
             if [[ "$labels" == *"ADMIN_TESTS"* ]]; then
               echo "ADMIN_TESTS=true" >> $GITHUB_OUTPUT
             fi
-          elif [[ ${{ github.event_name }} == 'merge_group' ]]; then
-            #TODO: add ability to opt out via MINIMAL_TESTS
-            echo "UPGRADE_LIGHT_TESTS=true" >> $GITHUB_OUTPUT
           elif [[ ${{ github.event_name }} == 'schedule' ]]; then
             echo "UPGRADE_TESTS=true" >> $GITHUB_OUTPUT
             echo "UPGRADE_LIGHT_TESTS=true" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/execute_advanced_tests.yaml b/.github/workflows/execute_advanced_tests.yaml
index 77d6a2b6bd..e8a4683812 100644
--- a/.github/workflows/execute_advanced_tests.yaml
+++ b/.github/workflows/execute_advanced_tests.yaml
@@ -3,14 +3,6 @@ name: "TESTING:ADVANCED:E2E"
 on:
   workflow_dispatch:
     inputs:
-      e2e-admin-tests:
-        type: boolean
-        required: false
-        default: false
-      e2e-upgrade-test:
-        type: boolean
-        required: false
-        default: false
       e2e-stateful-upgrade-test:
         type: boolean
         required: false
@@ -19,10 +11,6 @@ on:
         type: boolean
         required: false
         default: false
-      e2e-upgrade-test-light:
-        type: boolean
-        required: false
-        default: false
       e2e-stateful-data-test:
         type: boolean
         required: false
@@ -37,69 +25,6 @@ on:
     - cron: "0 6 * * *"
 
 jobs:
-  e2e-admin-tests:
-    if: ${{ github.event.inputs.e2e-admin-tests == 'true' || github.event_name == 'schedule' }}
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    timeout-minutes: 120
-    steps:
-      - name: "Checkout Code"
-        uses: actions/checkout@v4
-
-      - name: Start Test
-        run: make start-e2e-admin-test
-
-      # use docker logs -f rather than docker attach to make sure we get the initial logs
-      - name: Watch Test
-        run: |
-          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
-          docker logs -f "${container_id}" &
-          exit $(docker wait "${container_id}")
-        
-      - name: Full Log Dump On Failure
-        if: failure()
-        run: |
-          make stop-localnet
-
-      - name: Notify Slack on Failure
-        if: failure() && github.event_name == 'schedule'
-        uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
-
-  e2e-upgrade-test:
-    if: ${{ github.event.inputs.e2e-upgrade-test == 'true' || github.event_name == 'schedule' }}
-    runs-on: buildjet-16vcpu-ubuntu-2204
-    timeout-minutes: 120
-    steps:
-      - name: "Checkout Code"
-        uses: actions/checkout@v4
-
-      - name: Start Test
-        run: make start-upgrade-test
-
-      - name: Watch Test
-        run: |
-          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
-          docker logs -f "${container_id}" &
-          exit $(docker wait "${container_id}")
-
-      - name: Full Log Dump On Failure
-        if: failure()
-        run: |
-          make stop-localnet
-
-      - name: Notify Slack on Failure
-        if: failure() && github.event_name == 'schedule'
-        uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
-
   e2e-stateful-upgrade-test:
     if: ${{ github.event.inputs.e2e-stateful-upgrade-test == 'true' || github.event_name == 'schedule' }}
     runs-on: buildjet-16vcpu-ubuntu-2204
@@ -131,37 +56,6 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
 
-  e2e-upgrade-test-light:
-    if: ${{ github.event.inputs.e2e-upgrade-test-light == 'true' }}
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    timeout-minutes: 120
-    steps:
-      - name: "Checkout Code"
-        uses: actions/checkout@v4
-
-      - name: Start Test
-        run: make start-upgrade-test-light
-
-      - name: Watch Test
-        run: |
-          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
-          docker logs -f "${container_id}" &
-          exit $(docker wait "${container_id}")
-
-      - name: Full Log Dump On Failure
-        if: failure()
-        run: |
-          make stop-localnet
-
-      - name: Notify Slack on Failure
-        if: failure() && github.event_name == 'schedule'
-        uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
-
   e2e-performance-test:
     if: ${{ github.event.inputs.e2e-performance-test == 'true' }}
     runs-on: buildjet-4vcpu-ubuntu-2204
diff --git a/Makefile b/Makefile
index 6fd58f0484..81a7220c2d 100644
--- a/Makefile
+++ b/Makefile
@@ -28,11 +28,7 @@ export DOCKER_BUILDKIT := 1
 
 # parameters for localnet docker compose files
 # set defaults to empty to prevent docker warning
-export LOCALNET_MODE
 export E2E_ARGS := $(E2E_ARGS)
-export UPGRADE_HEIGHT
-export ZETACORED_IMPORT_GENESIS_DATA
-export ZETACORED_START_PERIOD := 30s
 
 clean: clean-binaries clean-dir clean-test-dir clean-coverage
 
@@ -217,7 +213,7 @@ start-localnet-skip-build:
 
 # stop-localnet should include all profiles so other containers are also removed
 stop-localnet:
-	cd contrib/localnet/ && $(DOCKER) compose --profile eth2 --profile stress --profile upgrade down --remove-orphans
+	cd contrib/localnet/ && $(DOCKER) compose --profile all --remove-orphans
 
 ###############################################################################
 ###                         E2E tests               						###
diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml
index cbfc17ff0f..a7dbadd9ea 100644
--- a/contrib/localnet/docker-compose.yml
+++ b/contrib/localnet/docker-compose.yml
@@ -84,6 +84,7 @@ services:
     hostname: zetacore2
     profiles:
       - stress
+      - all
     networks:
       mynetwork:
         ipv4_address: 172.20.0.13
@@ -100,6 +101,7 @@ services:
     hostname: zetacore3
     profiles:
       - stress
+      - all
     networks:
       mynetwork:
         ipv4_address: 172.20.0.14
@@ -148,6 +150,7 @@ services:
     hostname: zetaclient2
     profiles:
       - stress
+      - all
     networks:
       mynetwork:
         ipv4_address: 172.20.0.23
@@ -165,6 +168,7 @@ services:
     hostname: zetaclient3
     profiles:
       - stress
+      - all
     networks:
       mynetwork:
         ipv4_address: 172.20.0.24
@@ -194,6 +198,7 @@ services:
     hostname: eth2
     profiles:
       - eth2
+      - all
     platform: linux/amd64
     ports:
       - "8546:8545"
@@ -245,6 +250,7 @@ services:
     hostname: upgrade-host
     profiles:
       - upgrade
+      - all
     entrypoint: ["/root/start-upgrade-host.sh"]
     networks:
       mynetwork:
@@ -259,6 +265,7 @@ services:
     hostname: upgrade-orchestrator
     profiles:
       - upgrade
+      - all
     entrypoint: ["/root/start-upgrade-orchestrator.sh"]
     networks:
       mynetwork:

From 2133f2701fc7a8c0d6752e9118b7bc14173b334a Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 10:50:24 -0700
Subject: [PATCH 09/16] fix log dump

---
 .github/workflows/reusable-e2e.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
index a66f72d10a..da49a57370 100644
--- a/.github/workflows/reusable-e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -91,7 +91,14 @@ jobs:
       - name: Full Log Dump On Failure
         if: failure()
         run: |
-          make stop-localnet
+          cd contrib/localnet && docker compose --profile all logs --timestamps > /tmp/logs.txt
+
+      - name: Upload Logs On Failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ inputs.make-target }}-docker-logs
+          path: /tmp/logs.txt
 
       - name: Notify Slack on Failure
         if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'

From d49b7825708fcb68b0170b59abc9ea7371f0c4a1 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 10:53:44 -0700
Subject: [PATCH 10/16] or schedule

---
 .github/workflows/reusable-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
index da49a57370..79e595501a 100644
--- a/.github/workflows/reusable-e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -101,7 +101,7 @@ jobs:
           path: /tmp/logs.txt
 
       - name: Notify Slack on Failure
-        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
+        if: failure() && (github.event_name == 'push' && github.ref == 'refs/heads/develop') || github.event_name == 'schedule'
         uses: 8398a7/action-slack@v3
         with:
           status: ${{ job.status }}

From 08db0e4dc868f8ce1b59f2a731d4362322b53b24 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 10:56:23 -0700
Subject: [PATCH 11/16] fix stop-localnet

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 81a7220c2d..177f064722 100644
--- a/Makefile
+++ b/Makefile
@@ -213,7 +213,7 @@ start-localnet-skip-build:
 
 # stop-localnet should include all profiles so other containers are also removed
 stop-localnet:
-	cd contrib/localnet/ && $(DOCKER) compose --profile all --remove-orphans
+	cd contrib/localnet/ && $(DOCKER) compose --profile all down --remove-orphans
 
 ###############################################################################
 ###                         E2E tests               						###

From eca8bca17c481ab1fb5d835032d497dac719d34c Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 11:00:45 -0700
Subject: [PATCH 12/16] add .env file

---
 contrib/localnet/.env | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 contrib/localnet/.env

diff --git a/contrib/localnet/.env b/contrib/localnet/.env
new file mode 100644
index 0000000000..4f26b4a624
--- /dev/null
+++ b/contrib/localnet/.env
@@ -0,0 +1,5 @@
+LOCALNET_MODE=setup-only
+E2E_ARGS=
+UPGRADE_HEIGHT=
+ZETACORED_IMPORT_GENESIS_DATA=
+ZETACORED_START_PERIOD=30s
\ No newline at end of file

From 0a4fda653014f4bcd55d4631aec86a082a2d5f46 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 11:07:37 -0700
Subject: [PATCH 13/16] adjust defaults

---
 contrib/localnet/.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/localnet/.env b/contrib/localnet/.env
index 4f26b4a624..596d50194f 100644
--- a/contrib/localnet/.env
+++ b/contrib/localnet/.env
@@ -1,4 +1,4 @@
-LOCALNET_MODE=setup-only
+LOCALNET_MODE=
 E2E_ARGS=
 UPGRADE_HEIGHT=
 ZETACORED_IMPORT_GENESIS_DATA=

From f9c1d46078bbf117311cdfe0d7617c5aab266e23 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 11:13:01 -0700
Subject: [PATCH 14/16] alternative default syntax

---
 contrib/localnet/.env               |  5 -----
 contrib/localnet/docker-compose.yml | 12 ++++++------
 2 files changed, 6 insertions(+), 11 deletions(-)
 delete mode 100644 contrib/localnet/.env

diff --git a/contrib/localnet/.env b/contrib/localnet/.env
deleted file mode 100644
index 596d50194f..0000000000
--- a/contrib/localnet/.env
+++ /dev/null
@@ -1,5 +0,0 @@
-LOCALNET_MODE=
-E2E_ARGS=
-UPGRADE_HEIGHT=
-ZETACORED_IMPORT_GENESIS_DATA=
-ZETACORED_START_PERIOD=30s
\ No newline at end of file
diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml
index a7dbadd9ea..880b6de8ad 100644
--- a/contrib/localnet/docker-compose.yml
+++ b/contrib/localnet/docker-compose.yml
@@ -50,7 +50,7 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
-      start_period: ${ZETACORED_START_PERIOD}
+      start_period: ${ZETACORED_START_PERIOD-30s}
       start_interval: 1s
     networks:
       mynetwork:
@@ -59,7 +59,7 @@ services:
     environment:
       - HOTKEY_BACKEND=file
       - HOTKEY_PASSWORD=password # test purposes only
-      - ZETACORED_IMPORT_GENESIS_DATA=${ZETACORED_IMPORT_GENESIS_DATA}
+      - ZETACORED_IMPORT_GENESIS_DATA=${ZETACORED_IMPORT_GENESIS_DATA-}
     volumes:
       - ssh:/root/.ssh
       - ~/.zetacored/genesis_data:/root/genesis_data
@@ -238,9 +238,9 @@ services:
         ipv4_address: 172.20.0.2
     entrypoint: ["/work/start-zetae2e.sh", "local"]
     environment:
-      - LOCALNET_MODE=${LOCALNET_MODE}
-      - E2E_ARGS=${E2E_ARGS}
-      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT}
+      - LOCALNET_MODE=${LOCALNET_MODE-}
+      - E2E_ARGS=${E2E_ARGS-}
+      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
     volumes:
       - ssh:/root/.ssh
 
@@ -274,7 +274,7 @@ services:
       - zetacore0
       - upgrade-host
     environment:
-      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT}
+      - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-}
     volumes:
       - ssh:/root/.ssh
 volumes:

From ed620d87e58244cca8491e8ca876f45663cf0fa1 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 11:26:31 -0700
Subject: [PATCH 15/16] do not fail fast, use newer action versions

---
 .github/workflows/e2e.yml          | 1 +
 .github/workflows/reusable-e2e.yml | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index af85975403..e6405e214b 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -53,6 +53,7 @@ jobs:
   e2e:
     needs: matrix-conditionals
     strategy:
+      fail-fast: false
       matrix:
         include:
           - make-target: "start-e2e-test"
diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
index 79e595501a..2407598141 100644
--- a/.github/workflows/reusable-e2e.yml
+++ b/.github/workflows/reusable-e2e.yml
@@ -33,14 +33,14 @@ jobs:
       - uses: depot/use-containerd-snapshotter-action@v1
 
       - name: Login to Docker Hub registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: (github.event_name == 'push' && github.repository == 'zeta-chain/node') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'zeta-chain/node')
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_READ_ONLY }}
 
       - name: Login to github docker registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -95,7 +95,7 @@ jobs:
 
       - name: Upload Logs On Failure
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.make-target }}-docker-logs
           path: /tmp/logs.txt

From 7ece34edf392a8e0609d043eebc280689e5bf480 Mon Sep 17 00:00:00 2001
From: Alex Gartner <alexg@zetachain.com>
Date: Wed, 3 Jul 2024 12:04:46 -0700
Subject: [PATCH 16/16] rename to e2e-ok because status checks are based on job
 name

---
 .github/workflows/e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index e6405e214b..a63047890a 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -76,7 +76,7 @@ jobs:
       run: ${{ matrix.run }}
     secrets: inherit
   # this allows you to set a required status check
-  ok:
+  e2e-ok:
     runs-on: ubuntu-22.04
     needs: e2e
     if: always()