From d363a99e0cfb2f49f249888eb9537682c0a3959f Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Aug 2023 10:12:42 +0200 Subject: [PATCH] [ci] Add missing components to CI --- .gitignore | 2 + .gitlab-ci.yml | 181 ++++++++++++++++++++++++++++--------- .gitlab/pipeline/check.yml | 1 - 3 files changed, 141 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index b71c270d7368..f0ff64eba19a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ polkadot_argument_parsing .env **/._* +cargo_home +cargo_target_dir diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13a11b9bb4f3..bb4224e3445f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,9 @@ -# .gitlab-ci.yml +# polkadot-sdk | CI definitions (via GitLab CI) # -# substrate +# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions) # -# pipelines can be triggered manually in the web -# -# Currently the file is divided into subfiles. Each stage has a different file which -# can be found here: gitlab/pipeline/.yml -# -# Instead of YAML anchors "extends" is used. -# Useful links: -# https://docs.gitlab.com/ee/ci/yaml/index.html#extends -# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags -# -# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a -# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ -# -# my-example-job: -# stage: test # One of the stages listed below this job (required) -# image: paritytech/tools:latest # Any docker image (required) -# allow_failure: true # Allow the pipeline to continue if this job fails (default: false) -# needs: -# - job: test-linux # Any jobs that are required to run before this job (optional) -# variables: -# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) -# script: -# - echo "List of shell commands to run in your job" -# - echo "You can also just specify a script here, like so:" -# - ./gitlab/my_amazing_script.sh +# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding +# file which can be found here: .gitlab/pipeline/.yml stages: - check @@ -100,7 +77,7 @@ default: .kubernetes-env: image: "${CI_IMAGE}" before_script: - # - !reference [.job-switcher, before_script] + - !reference [.job-switcher, before_script] - !reference [.prepare-env, before_script] tags: - kubernetes-parity-build @@ -112,6 +89,8 @@ default: - rustup +nightly show - cargo +nightly --version +# collecting vars for pipeline stopper +# they will be used if the job fails .pipeline-stopper-vars: script: - !reference [.job-switcher, before_script] @@ -133,23 +112,21 @@ default: - !reference [.rust-info-script, script] - !reference [.rusty-cachier, before_script] - !reference [.pipeline-stopper-vars, script] - after_script: - - !reference [.rusty-cachier, after_script] tags: - linux-docker-vm-c2 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier -# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description +# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description .rusty-cachier: before_script: - # - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - # - rusty-cachier environment check --gracefully - # - $(rusty-cachier environment inject) - # - rusty-cachier project mtime - - echo tbd - after_script: - - echo tbd - # - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy + - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash + - mkdir -p cargo_home cargo_target_dir + - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home + - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir + - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} + + - git restore-mtime + - rusty-cachier --version + - rusty-cachier project touch-changed .common-refs: rules: @@ -233,6 +210,126 @@ include: # zombienet jobs - .gitlab/pipeline/zombienet.yml # # timestamp handler - # - project: parity/infrastructure/ci_cd/shared - # ref: v0.2 - # file: /common/timestamp.yml + - project: parity/infrastructure/ci_cd/shared + ref: v0.2 + file: /common/timestamp.yml + +# This job cancels the whole pipeline if any of provided jobs fail. +# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# to fail the pipeline as soon as possible to shorten the feedback loop. +.cancel-pipeline-template: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: on_failure + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PROJECT_NAME: "${CI_PROJECT_NAME}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + FAILED_JOB_URL: "${FAILED_JOB_URL}" + FAILED_JOB_NAME: "${FAILED_JOB_NAME}" + PR_NUM: "${PR_NUM}" + trigger: + project: "parity/infrastructure/ci_cd/pipeline-stopper" + branch: "as-improve" + +remove-cancel-pipeline-message: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PROJECT_NAME: "${CI_PROJECT_NAME}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + FAILED_JOB_URL: "https://gitlab.com" + FAILED_JOB_NAME: "nope" + PR_NUM: "${CI_COMMIT_REF_NAME}" + trigger: + project: "parity/infrastructure/ci_cd/pipeline-stopper" + +# need to copy jobs this way because otherwise gitlab will wait +# for all 3 jobs to finish instead of cancelling if one fails +cancel-pipeline-test-linux-stable1: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 1/3" + +cancel-pipeline-test-linux-stable2: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 2/3" + +cancel-pipeline-test-linux-stable3: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable 3/3" + +cancel-pipeline-test-linux-stable-additional-tests: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable-additional-tests" + +cancel-pipeline-test-linux-stable-slow: + extends: .cancel-pipeline-template + needs: + - job: "test-linux-stable-slow" + +cancel-pipeline-cargo-check-benches1: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-benches 1/2" + +cancel-pipeline-cargo-check-benches2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-benches 2/2" + +cancel-pipeline-test-linux-stable-int: + extends: .cancel-pipeline-template + needs: + - job: test-linux-stable-int + +cancel-pipeline-cargo-check-each-crate-1: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 1/6" + +cancel-pipeline-cargo-check-each-crate-2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 2/6" + +cancel-pipeline-cargo-check-each-crate-3: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 3/6" + +cancel-pipeline-cargo-check-each-crate-4: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 4/6" + +cancel-pipeline-cargo-check-each-crate-5: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 5/6" + +cancel-pipeline-cargo-check-each-crate-6: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 6/6" + +cancel-pipeline-cargo-check-each-crate-macos: + extends: .cancel-pipeline-template + needs: + - job: cargo-check-each-crate-macos + +cancel-pipeline-check-tracing: + extends: .cancel-pipeline-template + needs: + - job: check-tracing + +cancel-pipeline-cargo-clippy: + extends: .cancel-pipeline-template + needs: + - job: cargo-clippy diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 123e02a8cd1c..073a6de6266a 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -43,7 +43,6 @@ cargo-deny-licenses: script: - $CARGO_DENY_CMD --hide-inclusion-graph after_script: - # - !reference [.rusty-cachier, after_script] - echo "___The complete log is in the artifacts___" - $CARGO_DENY_CMD 2> deny.log - if [ $CI_JOB_STATUS != 'success' ]; then