fog-ledger - Helm chart improvements for consolidating fog instances #2607
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2018-2022 The MobileCoin Foundation | |
| # | |
| # MobileCoin Core projects - Build, deploy to development. | |
| name: Mobilecoin CD | |
| env: | |
| CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public | |
| DOCKER_ORG: mobilecoin | |
| RELEASE_5X_TAG: v5.0.3-dev | |
| on: | |
| pull_request: | |
| branches: | |
| - 'release/**' | |
| paths-ignore: | |
| - '**.md' | |
| push: | |
| branches: | |
| - 'feature/**' | |
| tags: | |
| - 'v[0-9]+*' | |
| paths-ignore: | |
| - '**.md' | |
| # don't run more than one at a time for a branch/tag | |
| concurrency: | |
| group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| # Ignore dependabot. We just need to 'if' the top level jobs. | |
| # Other jobs that 'need' these top level jobs will be skipped. | |
| jobs: | |
| ############################################ | |
| # Generate environment information | |
| ############################################ | |
| generate-metadata: | |
| if: ${{ ! startsWith(github.head_ref, 'dependabot/') }} | |
| name: 👾 Environment Info 👾 | |
| runs-on: [self-hosted, Linux, small] | |
| outputs: | |
| namespace: ${{ steps.meta.outputs.namespace }} | |
| tag: ${{ steps.meta.outputs.tag }} | |
| docker_tag: ${{ steps.meta.outputs.docker_tag }} | |
| docker_org: ${{ env.DOCKER_ORG }} | |
| chart_repo: ${{ env.CHART_REPO }} | |
| release_5x_tag: ${{ env.RELEASE_5X_TAG }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Generate version metadata | |
| id: meta | |
| shell: bash | |
| run: | | |
| .internal-ci/util/metadata.sh | |
| - name: 👾 Print Environment Details 👾 | |
| shell: bash | |
| env: | |
| CHART_REPO: ${{ env.CHART_REPO }} | |
| NAMESPACE: ${{ steps.meta.outputs.namespace }} | |
| VERSION: ${{ steps.meta.outputs.tag }} | |
| run: | | |
| .internal-ci/util/print_details.sh | |
| ######################################### | |
| # Build binaries | |
| ######################################### | |
| build-rust-hardware-projects: | |
| needs: | |
| - generate-metadata | |
| runs-on: [self-hosted, Linux, large-cd] | |
| container: | |
| image: mobilecoin/rust-sgx-base:v0.0.25 | |
| env: | |
| ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem | |
| MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem | |
| steps: | |
| - name: Checkout | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| uses: actions/checkout@v3 | |
| - name: Write environment values | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| env: | |
| ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }} | |
| MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }} | |
| run: | | |
| mkdir -p "${GITHUB_WORKSPACE}/.tmp" | |
| echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}" | |
| echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}" | |
| - name: Cache rust build binaries | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| id: rust_artifact_cache | |
| uses: ./.github/actions/mobilecoin-cache-rust-binaries | |
| with: | |
| cache_buster: ${{ secrets.CACHE_BUSTER }} | |
| - name: Cache cargo packages | |
| # We don't need cargo packages if we already have binaries. | |
| if: | | |
| steps.rust_artifact_cache.outputs.cache-hit != 'true' && | |
| ! contains(github.event.head_commit.message, '[skip build]') | |
| uses: ./.github/actions/mobilecoin-cache-cargo-package | |
| with: | |
| cache_buster: ${{ secrets.CACHE_BUSTER }} | |
| - name: Build rust hardware projects | |
| if: | | |
| steps.rust_artifact_cache.outputs.cache-hit != 'true' && | |
| ! contains(github.event.head_commit.message, '[skip build]') | |
| env: | |
| IAS_MODE: DEV | |
| SGX_MODE: HW | |
| RUST_BACKTRACE: full | |
| MOB_RELEASE: 1 | |
| CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} | |
| LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} | |
| VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} | |
| INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} | |
| run: | | |
| git config --global --add safe.directory '*' | |
| cargo build --release \ | |
| -p mc-admin-http-gateway \ | |
| -p mc-consensus-mint-client \ | |
| -p mc-consensus-service \ | |
| -p mc-fog-distribution \ | |
| -p mc-fog-ingest-client \ | |
| -p mc-fog-ingest-server \ | |
| -p mc-fog-ledger-server \ | |
| -p mc-fog-report-cli \ | |
| -p mc-fog-report-server \ | |
| -p mc-fog-sql-recovery-db \ | |
| -p mc-fog-test-client \ | |
| -p mc-fog-view-server \ | |
| -p mc-ledger-distribution \ | |
| -p mc-ledger-from-archive \ | |
| -p mc-ledger-migration \ | |
| -p mc-mobilecoind \ | |
| -p mc-mobilecoind-dev-faucet \ | |
| -p mc-mobilecoind-json \ | |
| -p mc-util-generate-sample-ledger \ | |
| -p mc-util-grpc-admin-tool \ | |
| -p mc-util-grpc-token-generator \ | |
| -p mc-util-keyfile \ | |
| -p mc-util-seeded-ed25519-key-gen \ | |
| -p mc-watcher | |
| - name: Copy artifacts to cache | |
| if: | | |
| steps.rust_artifact_cache.outputs.cache-hit != 'true' && | |
| ! contains(github.event.head_commit.message, '[skip build]') | |
| run: | | |
| mkdir -p rust_build_artifacts | |
| find target/release -maxdepth 1 -executable -type f -exec cp "{}" rust_build_artifacts/ \; | |
| find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" rust_build_artifacts/ \; | |
| - name: Create css measurements | |
| if: | | |
| steps.rust_artifact_cache.outputs.cache-hit != 'true' && | |
| ! contains(github.event.head_commit.message, '[skip build]') | |
| shell: bash | |
| run: | | |
| cd rust_build_artifacts | |
| for i in *.signed.so | |
| do | |
| css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/') | |
| sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css" | |
| done | |
| - name: Check artifacts | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| run: | | |
| ls -alR rust_build_artifacts | |
| - name: Upload artifacts | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: rust-binaries | |
| path: rust_build_artifacts/ | |
| build-go-projects: | |
| runs-on: [self-hosted, Linux, small] | |
| needs: | |
| - generate-metadata | |
| container: | |
| image: golang:1.18.5 | |
| steps: | |
| - name: Checkout | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| uses: actions/checkout@v3 | |
| - name: Add protobuf-compiler | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| run: | | |
| apt update | |
| apt install -y protobuf-compiler zstd | |
| - name: Cache go build binaries | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| id: go_artifact_cache | |
| uses: ./.github/actions/mobilecoin-cache-go-binaries | |
| with: | |
| cache_buster: ${{ secrets.CACHE_BUSTER }} | |
| - name: Build go-grpc-gateway | |
| if: | | |
| steps.go_artifact_cache.outputs.cache-hit != 'true' && | |
| ! contains(github.event.head_commit.message, '[skip build]') | |
| run: | | |
| cd go-grpc-gateway | |
| ./install_tools.sh | |
| ./build.sh | |
| mkdir -p ../go_build_artifacts | |
| cp go-grpc-gateway ../go_build_artifacts/ | |
| - name: check artifacts | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| run: | | |
| ls -alR go_build_artifacts | |
| - name: Upload Artifacts | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip build]') }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: go-binaries | |
| path: go_build_artifacts/ | |
| ######################################## | |
| # Create/Refresh base runtime image | |
| ######################################## | |
| docker-base: | |
| runs-on: [self-hosted, Linux, small] | |
| needs: | |
| - generate-metadata | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Generate Docker Tags | |
| id: docker_meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: ${{ env.DOCKER_ORG }}/runtime-base | |
| flavor: | | |
| latest=true | |
| tags: | | |
| type=sha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Publish to DockerHub | |
| id: docker_publish_dockerhub | |
| uses: docker/build-push-action@v4 | |
| with: | |
| build-args: | | |
| REPO_ORG=${{ env.DOCKER_ORG }} | |
| context: . | |
| file: .internal-ci/docker/Dockerfile.runtime-base | |
| labels: ${{ steps.docker_meta.outputs.labels }} | |
| push: true | |
| tags: ${{ steps.docker_meta.outputs.tags }} | |
| ######################################### | |
| # Build/Publish public artifacts | |
| ######################################### | |
| docker: | |
| runs-on: [self-hosted, Linux, small] | |
| needs: | |
| - build-go-projects | |
| - build-rust-hardware-projects | |
| - docker-base | |
| - generate-metadata | |
| strategy: | |
| matrix: | |
| image: | |
| - bootstrap-tools | |
| - fogingest | |
| - fog-ledger | |
| - fogreport | |
| - fog-test-client | |
| - fogview | |
| - go-grpc-gateway | |
| - node_hw | |
| - mobilecoind | |
| - watcher | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Cache rust build binaries | |
| id: rust_artifact_cache | |
| uses: ./.github/actions/mobilecoin-cache-rust-binaries | |
| with: | |
| cache_buster: ${{ secrets.CACHE_BUSTER }} | |
| - name: Cache go build binaries | |
| id: go_artifact_cache | |
| uses: ./.github/actions/mobilecoin-cache-go-binaries | |
| with: | |
| cache_buster: ${{ secrets.CACHE_BUSTER }} | |
| - name: Generate Docker Tags | |
| id: docker_meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: ${{ env.DOCKER_ORG }}/${{ matrix.image }} | |
| tags: ${{ needs.generate-metadata.outputs.docker_tag }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Get short SHA | |
| run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}" | |
| - name: Publish to DockerHub | |
| id: docker_publish_dockerhub | |
| uses: docker/build-push-action@v4 | |
| with: | |
| build-args: | | |
| REPO_ORG=${{ env.DOCKER_ORG }} | |
| BASE_TAG=${{ env.GH_SHORT_SHA }} | |
| RUST_BIN_PATH=rust_build_artifacts | |
| GO_BIN_PATH=go_build_artifacts | |
| cache-from: type=registry,ref=${{ env.DOCKER_ORG }}/${{ matrix.image }}:buildcache-${{ needs.generate-metadata.outputs.namespace }} | |
| cache-to: type=registry,ref=${{ env.DOCKER_ORG }}/${{ matrix.image }}:buildcache-${{ needs.generate-metadata.outputs.namespace }} | |
| context: . | |
| file: .internal-ci/docker/Dockerfile.${{ matrix.image }} | |
| labels: ${{ steps.docker_meta.outputs.labels }} | |
| push: true | |
| tags: ${{ steps.docker_meta.outputs.tags }} | |
| charts: | |
| runs-on: [self-hosted, Linux, small] | |
| needs: | |
| - docker | |
| - generate-metadata | |
| strategy: | |
| matrix: | |
| chart: | |
| - consensus-node | |
| - consensus-node-config | |
| - fog-ingest | |
| - fog-ingest-config | |
| - fog-test-client | |
| - mc-core-common-config | |
| - mc-core-dev-env-setup | |
| - mobilecoind | |
| - watcher | |
| - fog-report | |
| - fog-view | |
| - fog-ledger | |
| steps: | |
| - name: Checkout | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip charts]') }} | |
| uses: actions/checkout@v3 | |
| - name: Package and publish chart | |
| if: ${{ ! contains(github.event.head_commit.message, '[skip charts]') }} | |
| uses: mobilecoinofficial/gha-k8s-toolbox@v1 | |
| with: | |
| action: helm-publish | |
| chart_repo_username: ${{ secrets.HARBOR_USERNAME }} | |
| chart_repo_password: ${{ secrets.HARBOR_PASSWORD }} | |
| chart_repo: ${{ env.CHART_REPO }} | |
| chart_app_version: ${{ needs.generate-metadata.outputs.tag }} | |
| chart_version: ${{ needs.generate-metadata.outputs.tag }} | |
| chart_path: .internal-ci/helm/${{ matrix.chart }} | |
| ################################################ | |
| # Bootstrap namespace to v5.0.3-dev from backup | |
| ################################################ | |
| bootstrap-v5-bv3: | |
| uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml | |
| needs: | |
| - generate-metadata | |
| with: | |
| block_version: 3 | |
| chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} | |
| namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| version: ${{ needs.generate-metadata.outputs.release_5x_tag }} | |
| secrets: inherit | |
| ############################################### | |
| # Deploy current version to namespace block v3 | |
| ############################################### | |
| deploy-current-bv3-release: | |
| uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml | |
| needs: | |
| - bootstrap-v5-bv3 | |
| - charts | |
| - generate-metadata | |
| with: | |
| block_version: 3 | |
| chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} | |
| docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }} | |
| ingest_color: blue | |
| namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| version: ${{ needs.generate-metadata.outputs.tag }} | |
| secrets: inherit | |
| test-current-bv3-release: | |
| uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml | |
| needs: | |
| - deploy-current-bv3-release | |
| - generate-metadata | |
| with: | |
| fog_distribution: false | |
| ingest_color: blue | |
| namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| testing_block_v0: false | |
| testing_block_v2: false | |
| testing_block_v3: true | |
| generate_and_submit_mint_config_tx_uses_json: true | |
| secrets: inherit | |
| ################################################# | |
| # Update current consensus to namespace block vX | |
| ################################################# | |
| # update-current-to-bv3: | |
| # uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml | |
| # needs: | |
| # - test-current-bv2-release | |
| # - generate-metadata | |
| # with: | |
| # block_version: 3 | |
| # chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} | |
| # namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| # version: ${{ needs.generate-metadata.outputs.tag }} | |
| # secrets: inherit | |
| # test-current-bv3-release: | |
| # uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml | |
| # needs: | |
| # - update-current-to-bv3 | |
| # - generate-metadata | |
| # with: | |
| # fog_distribution: false | |
| # ingest_color: blue | |
| # namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| # testing_block_v0: false | |
| # testing_block_v2: false | |
| # testing_block_v3: true | |
| # generate_and_submit_mint_config_tx_uses_json: true | |
| # secrets: inherit | |
| mobilecoin-cd-complete: | |
| # Dummy step for a standard GHA Check that won't change when we update the tests. | |
| runs-on: [self-hosted, Linux, small] | |
| needs: | |
| - test-current-bv3-release | |
| steps: | |
| - name: CD is Complete | |
| run: 'true' | |
| ############################################################### | |
| # Clean up deployments | |
| ############################################################### | |
| # we keep feature/* | |
| # run on tag | |
| # run on pr to release/* | |
| cleanup-after-tag: | |
| if: github.ref_type == 'tag' | |
| needs: | |
| - test-current-bv3-release | |
| - generate-metadata | |
| uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml | |
| with: | |
| namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| delete_namespace: true | |
| secrets: inherit | |
| cleanup-after-pr-to-release-branch: | |
| if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') | |
| needs: | |
| - test-current-bv3-release | |
| - generate-metadata | |
| uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml | |
| with: | |
| namespace: ${{ needs.generate-metadata.outputs.namespace }} | |
| delete_namespace: true | |
| secrets: inherit |