Make chaos and load workflow compatible with toml test input (#238)

smartcontractkit · Oct 30, 2023 · 4c2b17e · 4c2b17e
1 parent 8f6b610
commit 4c2b17e
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 95 deletions.
diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml
@@ -30,15 +30,15 @@ jobs:
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: Check if image exists
         id: check-image
-        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           repository: chainlink
           tag: ${{ github.sha }}
           AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
       - name: Build Image
         if: steps.check-image.outputs.exists == 'false'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         env:
           GH_TOKEN: ${{ github.token }}
         with:
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [ build-chainlink, build-test-image ]
     env:
-      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
+      SELECTED_NETWORKS: SIMULATED
       TEST_SUITE: chaos
       TEST_ARGS: -test.timeout 30m
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
@@ -115,7 +115,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: Run Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: make gomod
@@ -131,11 +131,14 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           CGO_ENABLED: "1"
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
 
       ## Run Cleanup if the job succeeds
       - name: cleanup
         if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@ce87f8986ca18336cc5015df75916c2ec0a7c4b3 # v2.1.2
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           triggered_by: ${{ env.TEST_TRIGGERED_BY }}
 
@@ -151,7 +154,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [ build-chainlink, build-test-image ]
     env:
-      SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2
+      SELECTED_NETWORKS: SIMULATED
       TEST_SUITE: load
       TEST_ARGS: -test.timeout 1h
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
@@ -160,8 +163,8 @@ jobs:
       TEST_LOG_LEVEL: debug
       DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
       GH_TOKEN: ${{ github.token }}
-      TEST_LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }}
-      TEST_LOKI_URL: ${{ secrets.LOKI_URL }}
+      TEST_LOKI_TENANT_ID: ${{ secrets.TEST_LOKI_TENANT_ID }}
+      TEST_LOKI_URL: ${{ secrets.TEST_LOKI_URL }}
     steps:
       - name: Collect Metrics
         id: collect-gha-metrics
@@ -174,7 +177,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: Run Load With Chaos Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           test_command_to_run: make test_need_operator_assets && cd ./integration-tests/ccip-tests && go test -timeout 1h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: make gomod
@@ -190,10 +193,13 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           CGO_ENABLED: "1"
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
 
       ## Run Cleanup if the job succeeds
       - name: cleanup
         if: always()
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@ce87f8986ca18336cc5015df75916c2ec0a7c4b3 # v2.1.2
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           triggered_by: ${{ env.TEST_TRIGGERED_BY }}
diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
@@ -5,58 +5,9 @@ on:
       - ccip-develop
   workflow_dispatch:
     inputs:
-      existing-deployment:
-        description: 'Run load tests on already deployed contracts and jobs'
+      base64_test_input : # base64 encoded toml for test input
+        description: 'Base64 encoded toml test input'
         required: false
-        default: false
-        type: boolean
-      networks:
-        description: Network pairs to run tests on (comma separated list)
-        type: choice
-        default: "SIMULATED_1,SIMULATED_2"
-        options:
-          - "SIMULATED_1,SIMULATED_2"
-          - "OPTIMISM_GOERLI,SEPOLIA,GOERLI,AVALANCHE_FUJI,ARBITRUM_GOERLI,MUMBAI"
-      numberOfChains:
-        description: Number of networks to run tests on
-        type: choice
-        default: "2"
-        options:
-          - "2"
-          - "3"
-          - "4"
-          - "5"
-          - "6"
-      chainlinkImage:
-        description: Container image location for the Chainlink nodes
-        required: false
-        default: ""
-      chainlinkVersion:
-        description: Container image version for the Chainlink nodes
-        required: false
-        default: ""
-      chainlinkTestVersion:
-        description: Container image version for the Chainlink tests
-        required: false
-        default: ""
-      testDuration:
-        description: Duration of the test (time string)
-        required: false
-        default: 15m
-      send_request_rate:
-        description: Number of ccip-send requests to trigger per unit time duration
-        required: true
-        default: 2
-        type: number
-      send_request_rate_duration:
-        description: Time duration unit for ccip-send request rate
-        required: false
-        default: 1s
-      slackMemberID:
-        description: Slack Member ID (Not your @)
-        required: true
-        default: U01A2B2C3D4
-        type: string
 
 # Only run 1 of this workflow at a time per PR
 concurrency:
@@ -84,15 +35,15 @@ jobs:
         uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
       - name: Check if image exists
         id: check-image
-        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         with:
           repository: chainlink
           tag: ${{ env.INPUT_CHAINLINK_VERSION }}
           AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
       - name: Build Image
         if: steps.check-image.outputs.exists == 'false'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         env:
           GH_TOKEN: ${{ github.token }}
         with:
@@ -155,37 +106,19 @@ jobs:
       SLACK_USER: ${{ inputs.slackMemberID }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
-      SELECTED_NETWORKS: SIMULATED,${{ inputs.networks }}
+      SELECTED_NETWORKS: SIMULATED
       EVM_KEYS: ${{ secrets.QA_EVM_KEYS }} # replace it with input parameter once debug log is updated to not print the key
-      CCIP_TEST_DURATION: ${{ inputs.testDuration }}
-      CCIP_PHASE_VALIDATION_TIMEOUT: ${{ inputs.phase_time_out }}
-      CCIP_LOAD_TEST_RATE: ${{ inputs.send_request_rate }}
-      CCIP_LOAD_TEST_RATEUNIT: ${{ inputs.send_request_rate_duration }}
-      CCIP_MSG_TYPE: WithoutToken
-      CCIP_CHAINLINK_NODE_FUNDING: 20
-      CCIP_TESTS_ON_EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }}
-      CCIP_NO_OF_NETWORKS: ${{ inputs.numberOfChains }}
-      CCIP_KEEP_ENV_TTL: 5h
-      CCIP_REUSE_CONTRACTS: True
       TEST_LOG_LEVEL: info
       REF_NAME: ${{ github.head_ref || github.ref_name }}
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests
       GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }}
-      TEST_GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }}
       GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }}
-      TEST_GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }}
       OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }}
-      TEST_OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }}
       OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }}
-      TEST_OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }}
       SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }}
-      TEST_SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }}
       SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }}
-      TEST_SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }}
       AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }}
-      TEST_AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }}
       AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }}
-      TEST_AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }}
     steps:
       - name: Collect Metrics
         id: collect-gha-metrics
@@ -203,8 +136,8 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]; then
-            echo "SELECTED_NETWORKS=SIMULATED,SIMULATED_1,SIMULATED_2" >> $GITHUB_ENV
-            echo "CCIP_NO_OF_NETWORKS=2" >> $GITHUB_ENV
+            echo "BASE64_TEST_CONFIG_OVERRIDE= ${{ inputs.base64_test_input }}" >> $GITHUB_ENV
+            echo "TEST_BASE64_TEST_CONFIG_OVERRIDE= ${{ inputs.base64_test_input }}" >> $GITHUB_ENV
           fi
           if [[ "${{ env.INPUT_CHAINLINK_IMAGE }}" == "" || "${{ env.INPUT_CHAINLINK_VERSION }}" == "" ]]; then
             echo "INPUT_CHAINLINK_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink" >> $GITHUB_ENV
@@ -213,28 +146,26 @@ jobs:
           if [[ "${{ env.INPUT_CHAINLINK_TEST_VERSION }}" == "" ]]; then
             echo "INPUT_CHAINLINK_TEST_VERSION=${{ github.sha }}" >> $GITHUB_ENV
           fi
-          if [[ "${{ env.CCIP_NO_OF_NETWORKS }}" != "2" ]]; then
-            echo "RR_MEM=8Gi" >> $GITHUB_ENV
-            echo "RR_CPU=4" >> $GITHUB_ENV
-          fi
       - name: step summary
         shell: bash
-        env:
-          EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }}
         run: |
           echo "ENV_JOB_IMAGE=${{ env.ENV_JOB_IMAGE_BASE }}:${{ env.INPUT_CHAINLINK_TEST_VERSION }}" >> $GITHUB_ENV
           echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
           echo "\`${{ env.INPUT_CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
           echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
           echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "### test input override :link:" >>$GITHUB_STEP_SUMMARY
+          echo "$(base64 -d <<< "${{ inputs.base64_test_input }}")" >>$GITHUB_STEP_SUMMARY
       - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
         env:
           TEST_SUITE: load
           TEST_ARGS: -test.timeout 900h
           DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
-          TEST_LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }}
-          TEST_LOKI_URL: ${{ secrets.LOKI_URL }}
+          TEST_LOKI_TENANT_ID: ${{ secrets.TEST_LOKI_TENANT_ID }}
+          TEST_LOKI_URL: ${{ secrets.TEST_LOKI_URL }}
+          RR_MEM: 8Gi
+          RR_CPU: 4
         with:
           test_command_to_run: make test_need_operator_assets && cd ./integration-tests/ccip-tests && go test -v -timeout 900h -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load  2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -246,3 +177,6 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           artifacts_location: ./integration-tests/load/logs/payload_ccip.json
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
+          cache_restore_only: "true"
diff --git a/integration-tests/ccip-tests/Makefile b/integration-tests/ccip-tests/Makefile
@@ -1,4 +1,3 @@
-include test-config.env
 ## To Override the default config,
 # example usage: make override_config override_toml=../config/config.toml env=../.env
 .PHONY: override_config
@@ -10,7 +9,8 @@ override_config:
 
 # example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos
 .PHONY: test_load_ccip
-test_load_ccip:
+include test-config.env
+test_load_ccip: override_config
 	source ./testconfig/override/$(env) && \
     CHAINLINK_IMAGE=$(image)  \
     CHAINLINK_VERSION=$(tag)  \
@@ -25,6 +25,7 @@ test_load_ccip:
 # To run the test in besu simulated network, use the following command:
 # example usage: make test_smoke_ccip image=chainlink-ccip tag=latest testimage="" testname=TestSmokeCCIPForBidirectionalLane override_toml="" env=test-config.env
 .PHONY: test_smoke_ccip
+include test-config.env
 test_smoke_ccip: override_config
 	source ./testconfig/override/$(env) && \
     CHAINLINK_IMAGE=$(image)  \

diff --git a/integration-tests/ccip-tests/testconfig/tomls/default.toml b/integration-tests/ccip-tests/testconfig/tomls/default.toml
@@ -3,6 +3,7 @@
 [CCIP]
 # all variables to set up the test environment
 [CCIP.Env]
+TTL = '5h'
 # networks between which lanes will be set up and  the messages will be sent
 # if more than 2 networks are specified, then lanes will be set up between all possible pairs of networks
 # default value is ['SIMULATED_1', 'SIMULATED_2'] which means that test will create two private geth networks from scratch and set up lanes between them

diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go
@@ -661,6 +661,21 @@ func CCIPDefaultTestSetUp(
 				LocalCluster: local,
 			}
 		} else {
+			envConfig := &environment.Config{
+				NamespacePrefix: envName,
+				Test:            t,
+			}
+			if inputs.EnvInput.TTL != nil {
+				envConfig.TTL = inputs.EnvInput.TTL.Duration()
+			}
+
+			if inputs.TestGroupInput.TestDuration != nil {
+				approxDur := inputs.TestGroupInput.TestDuration.Duration() + 1*time.Hour
+				if envConfig.TTL < approxDur {
+					envConfig.TTL = approxDur
+				}
+			}
+
 			// deploy the env if configureCLNode is true
 			k8Env = DeployEnvironments(t, envConfig, inputs)
 			ccipEnv = &actions.CCIPTestEnv{K8Env: k8Env}