diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml index ba92c1d470..8e652bb523 100644 --- a/.github/workflows/ccip-chaos-tests.yml +++ b/.github/workflows/ccip-chaos-tests.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Check if image exists id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: repository: chainlink tag: ${{ github.sha }} @@ -38,7 +38,7 @@ jobs: AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Build Image if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 env: GH_TOKEN: ${{ github.token }} with: @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest needs: [ build-chainlink, build-test-image ] env: - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 + SELECTED_NETWORKS: SIMULATED TEST_SUITE: chaos TEST_ARGS: -test.timeout 30m CHAINLINK_COMMIT_SHA: ${{ github.sha }} @@ -115,7 +115,7 @@ jobs: - name: Checkout the repo uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Run Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: make gomod @@ -131,11 +131,14 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} CGO_ENABLED: "1" + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" ## Run Cleanup if the job succeeds - name: cleanup if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@ce87f8986ca18336cc5015df75916c2ec0a7c4b3 # v2.1.2 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: triggered_by: ${{ env.TEST_TRIGGERED_BY }} @@ -151,7 +154,7 @@ jobs: runs-on: ubuntu-latest needs: [ build-chainlink, build-test-image ] env: - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 + SELECTED_NETWORKS: SIMULATED TEST_SUITE: load TEST_ARGS: -test.timeout 1h CHAINLINK_COMMIT_SHA: ${{ github.sha }} @@ -160,8 +163,8 @@ jobs: TEST_LOG_LEVEL: debug DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable GH_TOKEN: ${{ github.token }} - TEST_LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }} - TEST_LOKI_URL: ${{ secrets.LOKI_URL }} + TEST_LOKI_TENANT_ID: ${{ secrets.TEST_LOKI_TENANT_ID }} + TEST_LOKI_URL: ${{ secrets.TEST_LOKI_URL }} steps: - name: Collect Metrics id: collect-gha-metrics @@ -174,7 +177,7 @@ jobs: - name: Checkout the repo uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Run Load With Chaos Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: test_command_to_run: make test_need_operator_assets && cd ./integration-tests/ccip-tests && go test -timeout 1h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: make gomod @@ -190,10 +193,13 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} CGO_ENABLED: "1" + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" ## Run Cleanup if the job succeeds - name: cleanup if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@ce87f8986ca18336cc5015df75916c2ec0a7c4b3 # v2.1.2 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: triggered_by: ${{ env.TEST_TRIGGERED_BY }} diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml index 8b9d7e8559..4fea1bd5f6 100644 --- a/.github/workflows/ccip-load-tests.yml +++ b/.github/workflows/ccip-load-tests.yml @@ -5,58 +5,9 @@ on: - ccip-develop workflow_dispatch: inputs: - existing-deployment: - description: 'Run load tests on already deployed contracts and jobs' + base64_test_input : # base64 encoded toml for test input + description: 'Base64 encoded toml test input' required: false - default: false - type: boolean - networks: - description: Network pairs to run tests on (comma separated list) - type: choice - default: "SIMULATED_1,SIMULATED_2" - options: - - "SIMULATED_1,SIMULATED_2" - - "OPTIMISM_GOERLI,SEPOLIA,GOERLI,AVALANCHE_FUJI,ARBITRUM_GOERLI,MUMBAI" - numberOfChains: - description: Number of networks to run tests on - type: choice - default: "2" - options: - - "2" - - "3" - - "4" - - "5" - - "6" - chainlinkImage: - description: Container image location for the Chainlink nodes - required: false - default: "" - chainlinkVersion: - description: Container image version for the Chainlink nodes - required: false - default: "" - chainlinkTestVersion: - description: Container image version for the Chainlink tests - required: false - default: "" - testDuration: - description: Duration of the test (time string) - required: false - default: 15m - send_request_rate: - description: Number of ccip-send requests to trigger per unit time duration - required: true - default: 2 - type: number - send_request_rate_duration: - description: Time duration unit for ccip-send request rate - required: false - default: 1s - slackMemberID: - description: Slack Member ID (Not your @) - required: true - default: U01A2B2C3D4 - type: string # Only run 1 of this workflow at a time per PR concurrency: @@ -84,7 +35,7 @@ jobs: uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Check if image exists id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 with: repository: chainlink tag: ${{ env.INPUT_CHAINLINK_VERSION }} @@ -92,7 +43,7 @@ jobs: AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Build Image if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 env: GH_TOKEN: ${{ github.token }} with: @@ -155,37 +106,19 @@ jobs: SLACK_USER: ${{ inputs.slackMemberID }} SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} - SELECTED_NETWORKS: SIMULATED,${{ inputs.networks }} + SELECTED_NETWORKS: SIMULATED EVM_KEYS: ${{ secrets.QA_EVM_KEYS }} # replace it with input parameter once debug log is updated to not print the key - CCIP_TEST_DURATION: ${{ inputs.testDuration }} - CCIP_PHASE_VALIDATION_TIMEOUT: ${{ inputs.phase_time_out }} - CCIP_LOAD_TEST_RATE: ${{ inputs.send_request_rate }} - CCIP_LOAD_TEST_RATEUNIT: ${{ inputs.send_request_rate_duration }} - CCIP_MSG_TYPE: WithoutToken - CCIP_CHAINLINK_NODE_FUNDING: 20 - CCIP_TESTS_ON_EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }} - CCIP_NO_OF_NETWORKS: ${{ inputs.numberOfChains }} - CCIP_KEEP_ENV_TTL: 5h - CCIP_REUSE_CONTRACTS: True TEST_LOG_LEVEL: info REF_NAME: ${{ github.head_ref || github.ref_name }} ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }} - TEST_GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }} GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }} - TEST_GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }} OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }} - TEST_OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }} OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }} - TEST_OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }} SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }} - TEST_SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }} SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} - TEST_SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }} - TEST_AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }} AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }} - TEST_AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }} steps: - name: Collect Metrics id: collect-gha-metrics @@ -203,8 +136,8 @@ jobs: shell: bash run: | if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]; then - echo "SELECTED_NETWORKS=SIMULATED,SIMULATED_1,SIMULATED_2" >> $GITHUB_ENV - echo "CCIP_NO_OF_NETWORKS=2" >> $GITHUB_ENV + echo "BASE64_TEST_CONFIG_OVERRIDE= ${{ inputs.base64_test_input }}" >> $GITHUB_ENV + echo "TEST_BASE64_TEST_CONFIG_OVERRIDE= ${{ inputs.base64_test_input }}" >> $GITHUB_ENV fi if [[ "${{ env.INPUT_CHAINLINK_IMAGE }}" == "" || "${{ env.INPUT_CHAINLINK_VERSION }}" == "" ]]; then echo "INPUT_CHAINLINK_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink" >> $GITHUB_ENV @@ -213,28 +146,26 @@ jobs: if [[ "${{ env.INPUT_CHAINLINK_TEST_VERSION }}" == "" ]]; then echo "INPUT_CHAINLINK_TEST_VERSION=${{ github.sha }}" >> $GITHUB_ENV fi - if [[ "${{ env.CCIP_NO_OF_NETWORKS }}" != "2" ]]; then - echo "RR_MEM=8Gi" >> $GITHUB_ENV - echo "RR_CPU=4" >> $GITHUB_ENV - fi - name: step summary shell: bash - env: - EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }} run: | echo "ENV_JOB_IMAGE=${{ env.ENV_JOB_IMAGE_BASE }}:${{ env.INPUT_CHAINLINK_TEST_VERSION }}" >> $GITHUB_ENV echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.INPUT_CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY + echo "### test input override :link:" >>$GITHUB_STEP_SUMMARY + echo "$(base64 -d <<< "${{ inputs.base64_test_input }}")" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13 env: TEST_SUITE: load TEST_ARGS: -test.timeout 900h DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable - TEST_LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }} - TEST_LOKI_URL: ${{ secrets.LOKI_URL }} + TEST_LOKI_TENANT_ID: ${{ secrets.TEST_LOKI_TENANT_ID }} + TEST_LOKI_URL: ${{ secrets.TEST_LOKI_URL }} + RR_MEM: 8Gi + RR_CPU: 4 with: test_command_to_run: make test_need_operator_assets && cd ./integration-tests/ccip-tests && go test -v -timeout 900h -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -246,3 +177,6 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} artifacts_location: ./integration-tests/load/logs/payload_ccip.json + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" \ No newline at end of file diff --git a/integration-tests/ccip-tests/Makefile b/integration-tests/ccip-tests/Makefile index a2a8531cb2..e151dde2a4 100644 --- a/integration-tests/ccip-tests/Makefile +++ b/integration-tests/ccip-tests/Makefile @@ -1,4 +1,3 @@ -include test-config.env ## To Override the default config, # example usage: make override_config override_toml=../config/config.toml env=../.env .PHONY: override_config @@ -10,7 +9,8 @@ override_config: # example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos .PHONY: test_load_ccip -test_load_ccip: +include test-config.env +test_load_ccip: override_config source ./testconfig/override/$(env) && \ CHAINLINK_IMAGE=$(image) \ CHAINLINK_VERSION=$(tag) \ @@ -25,6 +25,7 @@ test_load_ccip: # To run the test in besu simulated network, use the following command: # example usage: make test_smoke_ccip image=chainlink-ccip tag=latest testimage="" testname=TestSmokeCCIPForBidirectionalLane override_toml="" env=test-config.env .PHONY: test_smoke_ccip +include test-config.env test_smoke_ccip: override_config source ./testconfig/override/$(env) && \ CHAINLINK_IMAGE=$(image) \ diff --git a/integration-tests/ccip-tests/testconfig/tomls/default.toml b/integration-tests/ccip-tests/testconfig/tomls/default.toml index 82fa803182..392e6ecc0a 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/default.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/default.toml @@ -3,6 +3,7 @@ [CCIP] # all variables to set up the test environment [CCIP.Env] +TTL = '5h' # networks between which lanes will be set up and the messages will be sent # if more than 2 networks are specified, then lanes will be set up between all possible pairs of networks # default value is ['SIMULATED_1', 'SIMULATED_2'] which means that test will create two private geth networks from scratch and set up lanes between them diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index 0e571260df..4b25dd5c86 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -661,6 +661,21 @@ func CCIPDefaultTestSetUp( LocalCluster: local, } } else { + envConfig := &environment.Config{ + NamespacePrefix: envName, + Test: t, + } + if inputs.EnvInput.TTL != nil { + envConfig.TTL = inputs.EnvInput.TTL.Duration() + } + + if inputs.TestGroupInput.TestDuration != nil { + approxDur := inputs.TestGroupInput.TestDuration.Duration() + 1*time.Hour + if envConfig.TTL < approxDur { + envConfig.TTL = approxDur + } + } + // deploy the env if configureCLNode is true k8Env = DeployEnvironments(t, envConfig, inputs) ccipEnv = &actions.CCIPTestEnv{K8Env: k8Env}