Feat/lw 11842 performance improvements #3328
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yq -y < std-unexploded.yml > workflows/std.yml | |
| name: STD | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| deploy-dev-preview: | |
| description: Deploy to dev-preview | |
| type: boolean | |
| required: true | |
| default: false | |
| deploy-dev-preprod: | |
| description: Deploy to dev-preprod | |
| type: boolean | |
| required: true | |
| default: false | |
| deploy-staging-preprod: | |
| description: Deploy to staging-preprod | |
| type: boolean | |
| required: true | |
| default: false | |
| deploy-staging-mainnet: | |
| description: Deploy to staging-mainnet | |
| type: boolean | |
| required: true | |
| default: false | |
| deploy-dev-mainnet: | |
| description: Deploy to dev-mainnet | |
| type: boolean | |
| required: true | |
| default: false | |
| deploy-local-network: | |
| description: Deploy to local-network | |
| type: boolean | |
| required: true | |
| default: false | |
| workflow_call: | |
| inputs: | |
| deploy-dev-preprod: | |
| type: boolean | |
| required: true | |
| deploy-staging-preprod: | |
| type: boolean | |
| required: true | |
| deploy-dev-mainnet: | |
| type: boolean | |
| required: true | |
| secrets: | |
| AWS_ACCESS_KEY: | |
| required: true | |
| AWS_SECRET_ACCESS_KEY: | |
| required: true | |
| SSH_PRIVATE_KEY: | |
| required: true | |
| pull_request: | |
| branches: | |
| - master | |
| - conway-era | |
| push: | |
| branches: | |
| - master | |
| - conway-era | |
| tags: | |
| - '@cardano-sdk/cardano-services**' | |
| env: | |
| # NIX_UPLOAD_CACHE: s3://lace-nix-cache?region=us-east-1 | |
| DISCOVERY_USER_NAME: gha-runner | |
| DISCOVERY_KNOWN_HOSTS_ENTRY: '65.109.126.156 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEOVVDZydvD+diYa6A3EtA3WGw5NfN0wv7ckQxa/fX1O' | |
| concurrency: | |
| group: std-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| discover: | |
| # Don’t run on PRs from forks (no access to secrets): | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| outputs: | |
| hits: ${{ steps.discovery.outputs.hits }} | |
| deployment-matrix: ${{ steps.deployment-matrix.outputs.deployment-matrix }} | |
| runs-on: [self-hosted, discovery] | |
| env: | |
| AWS_REGION: us-east-1 | |
| AWS_ROLE_ARN: arn:aws:iam::926093910549:role/lace-ci | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ env.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| # account is part of ecr url, thus part of `hits` output and needs to pass | |
| mask-aws-account-id: false | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Show commit | |
| # TODO: uncomment when nixbuild works well together with | |
| # nix daemon mode on hosted runners | |
| # - uses: nixbuild/nixbuild-action@v17 | |
| # with: | |
| # nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| # generate_summary_for: job | |
| shell: bash | |
| run: | | |
| echo commit: ${{ github.sha }} | |
| - name: Determine Deployment Matrix | |
| id: deployment-matrix | |
| run: | | |
| ( | |
| if [ "true" == ${{ inputs.deploy-dev-preview || (github.event_name == 'push' && github.ref_name == 'master') }} ] ; then | |
| echo '{"environment":"dev-preview", "target":"dev-preview@us-east-1", "url": "https://dev-preview.lw.iog.io/"}' | |
| fi | |
| if [ "true" == ${{ inputs.deploy-dev-preprod || false }} ] ; then | |
| echo '{"environment":"dev-preprod", "target":"dev-preprod@us-east-1@v2", "url": "https://dev-preprod.lw.iog.io/"}' | |
| fi | |
| if [ "true" == ${{ inputs.deploy-staging-mainnet || false }} ] ; then | |
| echo '{"environment":"staging-mainnet", "target":"staging-mainnet@eu-west-1@v2", "url": "https://staging-mainnet.lw.iog.io/"}' | |
| fi | |
| if [ "true" == ${{ inputs.deploy-staging-preprod || false }} ] ; then | |
| echo '{"environment":"staging-preprod", "target":"staging-preprod@us-east-1@v2", "url": "https://staging-preprod.lw.iog.io/"}' | |
| fi | |
| if [ "true" == ${{ inputs.deploy-dev-mainnet || false }} ] ; then | |
| echo '{"environment":"dev-mainnet", "target":"dev-mainnet@us-east-1", "url": "https://dev-mainnet.lw.iog.io/"}' | |
| fi | |
| if [ "true" == ${{ inputs.deploy-local-network || false }} ] ; then | |
| echo '{"environment":"local-network", "target":"local-network@us-east-1@v1", "url": "https://local-network.lw.iog.io/"}' | |
| fi | |
| ) | jq --slurp >deployment-matrix.json | |
| cat deployment-matrix.json | |
| # TODO: should we remove the trailing double quotes? | |
| echo "deployment-matrix=$(cat deployment-matrix.json | jq -c . | jq --raw-input)" >> "$GITHUB_OUTPUT" | |
| - uses: divnix/std-action/discover@main | |
| with: { ffBuildInstructions: true } | |
| id: discovery | |
| images: | |
| name: ${{ matrix.target.jobName }} | |
| runs-on: ubuntu-latest | |
| needs: discover | |
| env: | |
| AWS_REGION: us-east-1 | |
| AWS_ROLE_ARN: arn:aws:iam::926093910549:role/lace-ci | |
| # NIX_UPLOAD_CACHE: s3://lace-nix-cache?region=us-east-1 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| target: ${{ fromJSON(needs.discover.outputs.hits).oci-images && fromJSON(needs.discover.outputs.hits).oci-images.publish || fromJSON('["dummy-target"]') }} | |
| steps: | |
| - name: Configure AWS Credentials | |
| if: matrix.target != 'dummy-target' | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ env.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Login to Amazon ECR | |
| if: matrix.target != 'dummy-target' | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - uses: nixbuild/nix-quick-install-action@v25 | |
| if: matrix.target != 'dummy-target' | |
| - uses: nixbuild/nixbuild-action@v17 | |
| if: matrix.target != 'dummy-target' | |
| with: | |
| nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| generate_summary_for: job | |
| - uses: divnix/std-action/setup-discovery-ssh@main | |
| if: matrix.target != 'dummy-target' | |
| with: | |
| ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| user_name: ${{ env.DISCOVERY_USER_NAME }} | |
| ssh_known_hosts_entry: ${{ env.DISCOVERY_KNOWN_HOSTS_ENTRY }} | |
| - name: Show commit | |
| shell: bash | |
| run: | | |
| echo commit: ${{ github.sha }} | |
| - uses: divnix/std-action/run@main | |
| if: matrix.target != 'dummy-target' | |
| with: { ffBuildInstructions: true, remoteStore: 'ssh-ng://eu.nixbuild.net' } | |
| diff: | |
| needs: images | |
| name: Diff & Comment | |
| if: github.event_name == 'pull_request' && (github.base_ref == 'master' || github.base_ref == 'conway-era') | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: nixbuild/nix-quick-install-action@v25 | |
| - uses: nixbuild/nixbuild-action@v17 | |
| with: | |
| nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| generate_summary_for: job | |
| # Further steps assume AWS_PROFILE=lw, while the official action has no way to specify that profile: | |
| - name: Set up AWS credentials | |
| run: | | |
| mkdir -p ~/.aws | |
| cat <<EOF >~/.aws/credentials | |
| [lw] | |
| aws_access_key_id = ${{ secrets.AWS_ACCESS_KEY}} | |
| aws_secret_access_key = ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| EOF | |
| cat <<EOF >~/.aws/config | |
| [lw] | |
| region = us-east-1 | |
| EOF | |
| - uses: divnix/std-action/setup-discovery-ssh@main | |
| with: | |
| ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| user_name: ${{ env.DISCOVERY_USER_NAME }} | |
| ssh_known_hosts_entry: ${{ env.DISCOVERY_KNOWN_HOSTS_ENTRY }} | |
| - name: Generate the Diff | |
| run: | | |
| echo 'export K8S_USER=eks-devs' >.envrc.local | |
| nix develop -L --command bash -c ' | |
| set -euo pipefail | |
| export AWS_PROFILE="lw" | |
| export AWS_REGION="us-east-1" | |
| printf "" >pr-comment.md | |
| for target in \ | |
| "dev-preview@us-east-1" \ | |
| "dev-preprod@us-east-1@v2" \ | |
| "dev-mainnet@us-east-1" \ | |
| ; do | |
| nix run -L ".#cardano-services.${target}.plan" | tee k8s-plan.diff | |
| ( | |
| echo "<details>" | |
| echo "<summary><code>${target}</code> would change:</summary>" | |
| echo | |
| cat k8s-plan.diff \ | |
| | sed -r "s|^[^ +-].*|\`\`\`\n\n\0\n\n\`\`\`diff|g" \ | |
| | tail -n +3 \ | |
| | sed -r "s|^([^ +-].*)has changed(.*)|\1would change\2|g" | |
| echo "\`\`\`" | |
| echo "</details>" | |
| ) >>pr-comment.md | |
| done | |
| ' | |
| - name: Post Comment on the PR | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| COMMENT_MARKER: 'nix-helm-diff-777f3796-c80d-4d68-bf4f-8faad564f03f' | |
| run: | | |
| prNumber=$(cut -d/ -f1 <<<'${{ github.ref_name }}') | |
| # The `gh` command doesn’t return numeric comment ids, but instead node_ids, which don’t work with the regular API | |
| # Why the regular API? Because the `gh` command doesn’t support editing comments: | |
| existingCommentId=$( | |
| curl --fail-with-body -sSL \ | |
| -X GET \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${prNumber}/comments" \ | |
| | jq -r --arg commentMarker "$COMMENT_MARKER" 'first(.[] | select(.body | contains($commentMarker)) | .id)' | |
| ) | |
| if [ -z "$existingCommentId" ]; then | |
| ( cat pr-comment.md && echo "<!-- $COMMENT_MARKER -->" ; ) >gh-pr-comment-data.md | |
| gh pr comment "$prNumber" --body-file gh-pr-comment-data.md | |
| else | |
| jq --null-input --rawfile body pr-comment.md --arg marker "<!-- $COMMENT_MARKER -->" '{body: ($body + $marker)}' >curl-patch-data.json | |
| curl --fail-with-body -sSL \ | |
| -X PATCH \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/comments/${existingCommentId}" \ | |
| -d @curl-patch-data.json | |
| fi | |
| deploy: | |
| needs: [images, discover] | |
| concurrency: | |
| # Only one deployment at a time per environment, and wait for the previous one to finish: | |
| group: deploy-${{ matrix.environment }} | |
| cancel-in-progress: false | |
| if: fromJSON(needs.discover.outputs.deployment-matrix) != '[]' | |
| strategy: | |
| matrix: | |
| include: ${{ fromJSON(fromJSON(needs.discover.outputs.deployment-matrix)) }} | |
| name: Deploy (${{ matrix.environment }}) | |
| runs-on: ubuntu-22.04 | |
| environment: | |
| name: ${{ matrix.environment }} | |
| url: ${{ matrix.url }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: nixbuild/nix-quick-install-action@v25 | |
| - uses: nixbuild/nixbuild-action@v17 | |
| with: | |
| nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| generate_summary_for: job | |
| # Further steps assume AWS_PROFILE=lw, while the official action has no way to specify that profile: | |
| - name: Set up AWS credentials | |
| run: | | |
| mkdir -p ~/.aws | |
| cat <<EOF >~/.aws/credentials | |
| [lw] | |
| aws_access_key_id = ${{ secrets.AWS_ACCESS_KEY}} | |
| aws_secret_access_key = ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| EOF | |
| cat <<EOF >~/.aws/config | |
| [lw] | |
| region = us-east-1 | |
| EOF | |
| - uses: divnix/std-action/setup-discovery-ssh@main | |
| with: | |
| ssh_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| user_name: ${{ env.DISCOVERY_USER_NAME }} | |
| ssh_known_hosts_entry: ${{ env.DISCOVERY_KNOWN_HOSTS_ENTRY }} | |
| - name: Deploy to K8s | |
| run: | | |
| echo 'export K8S_USER=eks-devs' >.envrc.local | |
| nix develop -L --command bash -c ' | |
| set -euo pipefail | |
| export AWS_PROFILE="lw" | |
| export AWS_REGION="us-east-1" | |
| echo yes | nix run -L ".#cardano-services.${{ matrix.target }}.apply" | |
| ' |