From 754cfb15084756c4b4d1abf834064be7b17dc1df Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Wed, 17 Jan 2024 20:24:21 +0000 Subject: [PATCH] chore: Add wait for all workflow (#505) --- ...ansformations_aws_asset_inventory_free.yml | 2 + .../transformations_aws_compliance_free.yml | 2 + ...transformations_aws_compliance_premium.yml | 2 + .../workflows/transformations_aws_cost.yml | 2 + .../transformations_aws_data_resilience.yml | 2 + .../transformations_aws_encryption.yml | 130 ++++++++++++++++++ .../transformations_azure_compliance_free.yml | 2 + ...ansformations_azure_compliance_premium.yml | 2 + .../transformations_gcp_compliance_free.yml | 2 + ...transformations_gcp_compliance_premium.yml | 2 + .../transformations_k8s_compliance_free.yml | 2 + ...transformations_k8s_compliance_premium.yml | 2 + .../workflows/wait_for_required_workflows.yml | 30 ++++ .../workflows/wait_for_required_workflows.js | 66 +++++++++ 14 files changed, 248 insertions(+) create mode 100644 .github/workflows/transformations_aws_encryption.yml create mode 100644 .github/workflows/wait_for_required_workflows.yml create mode 100644 scripts/workflows/wait_for_required_workflows.js diff --git a/.github/workflows/transformations_aws_asset_inventory_free.yml b/.github/workflows/transformations_aws_asset_inventory_free.yml index 38be0e830..d1f3ddc97 100644 --- a/.github/workflows/transformations_aws_asset_inventory_free.yml +++ b/.github/workflows/transformations_aws_asset_inventory_free.yml @@ -8,6 +8,8 @@ on: - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_aws_compliance_free.yml b/.github/workflows/transformations_aws_compliance_free.yml index da1ad8c1e..1aab752dd 100644 --- a/.github/workflows/transformations_aws_compliance_free.yml +++ b/.github/workflows/transformations_aws_compliance_free.yml @@ -8,6 +8,8 @@ on: - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml index 9501a6fc9..a6cdb976a 100644 --- a/.github/workflows/transformations_aws_compliance_premium.yml +++ b/.github/workflows/transformations_aws_compliance_premium.yml @@ -8,6 +8,8 @@ on: - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_aws_cost.yml b/.github/workflows/transformations_aws_cost.yml index 8d5cc3ecb..10270598b 100644 --- a/.github/workflows/transformations_aws_cost.yml +++ b/.github/workflows/transformations_aws_cost.yml @@ -8,6 +8,8 @@ on: - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_aws_data_resilience.yml b/.github/workflows/transformations_aws_data_resilience.yml index ce9f8654e..a9647b983 100644 --- a/.github/workflows/transformations_aws_data_resilience.yml +++ b/.github/workflows/transformations_aws_data_resilience.yml @@ -8,6 +8,8 @@ on: - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_aws_encryption.yml b/.github/workflows/transformations_aws_encryption.yml new file mode 100644 index 000000000..f17b8041d --- /dev/null +++ b/.github/workflows/transformations_aws_encryption.yml @@ -0,0 +1,130 @@ +name: "Test AWS Encryption Policies" + +on: + pull_request: + paths: + - "transformations/aws/encryption/**" + - ".github/workflows/transformations_aws_encryption.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" + push: + branches: + - main + paths: + - "transformations/aws/encryption/**" + - ".github/workflows/transformations_aws_encryption.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/encryption + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-aws-encryption: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v2' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v5.0.1 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt seed --target dev-pg --profiles-dir ./tests + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt seed --target dev-pg --profiles-dir ./tests + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt seed --target dev-pg --profiles-dir ./tests + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_azure_compliance_free.yml b/.github/workflows/transformations_azure_compliance_free.yml index 60e5a0377..eaa003d55 100644 --- a/.github/workflows/transformations_azure_compliance_free.yml +++ b/.github/workflows/transformations_azure_compliance_free.yml @@ -8,6 +8,8 @@ on: - "transformations/azure/macros/**" - "transformations/azure/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_azure_compliance_premium.yml b/.github/workflows/transformations_azure_compliance_premium.yml index b248d4e92..7acf467ff 100644 --- a/.github/workflows/transformations_azure_compliance_premium.yml +++ b/.github/workflows/transformations_azure_compliance_premium.yml @@ -8,6 +8,8 @@ on: - "transformations/azure/macros/**" - "transformations/azure/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_gcp_compliance_free.yml b/.github/workflows/transformations_gcp_compliance_free.yml index c44c168da..2ce12fda3 100644 --- a/.github/workflows/transformations_gcp_compliance_free.yml +++ b/.github/workflows/transformations_gcp_compliance_free.yml @@ -8,6 +8,8 @@ on: - "transformations/gcp/macros/**" - "transformations/gcp/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_gcp_compliance_premium.yml b/.github/workflows/transformations_gcp_compliance_premium.yml index 5199c119b..41ff6caf6 100644 --- a/.github/workflows/transformations_gcp_compliance_premium.yml +++ b/.github/workflows/transformations_gcp_compliance_premium.yml @@ -8,6 +8,8 @@ on: - "transformations/gcp/macros/**" - "transformations/gcp/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_k8s_compliance_free.yml b/.github/workflows/transformations_k8s_compliance_free.yml index 76460b6a4..d3fed174d 100644 --- a/.github/workflows/transformations_k8s_compliance_free.yml +++ b/.github/workflows/transformations_k8s_compliance_free.yml @@ -8,6 +8,8 @@ on: - "transformations/k8s/macros/**" - "transformations/k8s/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/transformations_k8s_compliance_premium.yml b/.github/workflows/transformations_k8s_compliance_premium.yml index 7301723fe..89fd438e2 100644 --- a/.github/workflows/transformations_k8s_compliance_premium.yml +++ b/.github/workflows/transformations_k8s_compliance_premium.yml @@ -8,6 +8,8 @@ on: - "transformations/k8s/macros/**" - "transformations/k8s/models/**" - "transformations/macros/**" + - ".github/workflows/wait_for_required_workflows.yml" + - "scripts/workflows/wait_for_required_workflows.js" push: branches: - main diff --git a/.github/workflows/wait_for_required_workflows.yml b/.github/workflows/wait_for_required_workflows.yml new file mode 100644 index 000000000..fca2e1d5d --- /dev/null +++ b/.github/workflows/wait_for_required_workflows.yml @@ -0,0 +1,30 @@ +name: Wait for all required workflows to pass + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: + - main + +jobs: + wait_for_required_workflows: + timeout-minutes: 60 + name: wait-for-required-workflows + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v41 + - uses: actions/github-script@v7 + env: + FILES: ${{ steps.changed-files.outputs.all_modified_files }} + with: + script: | + const script = require('./scripts/workflows/wait_for_required_workflows.js') + await script({github, context}) diff --git a/scripts/workflows/wait_for_required_workflows.js b/scripts/workflows/wait_for_required_workflows.js new file mode 100644 index 000000000..a0ac58cc1 --- /dev/null +++ b/scripts/workflows/wait_for_required_workflows.js @@ -0,0 +1,66 @@ +module.exports = async ({github, context}) => { + const files = process.env.FILES.split(' ') + console.log(files) + if (files.length >= 300) { + // This is a GitHub limitation https://github.com/cloudquery/cloudquery/issues/2688 + throw new Error('Too many files changed. Skipping check. Please split your PR into multiple ones.') + } + + const path = require("path"); + const fs = require("fs"); + let now = new Date().getTime() + const deadline = now + 60 * 1000 * 50 + const matchesWorkflow = (file, action) => { + if (!file.startsWith('.github/workflows/')) { + return false + } + try { + const contents = fs.readFileSync(file, 'utf8'); + return contents.includes(`"${action}`) + } catch { + return false + } + } + const matchesFile = (action) => { + return files.some(file => file.startsWith(`${action}/`) || matchesWorkflow(file, action)) + } + + const testAll = files.includes("scripts/workflows/wait_for_required_workflows.js") || files.includes(".github/workflows/wait_for_required_workflows.yml") + const transformations = fs.readdirSync("transformations", {withFileTypes: true, recursive: true}).filter(dirent => dirent.isFile() && dirent.name === "dbt_project.yml").map(dirent => dirent.path) + let actions = transformations.filter(action => testAll || matchesFile(action)) + if (actions.length === 0) { + console.log("No actions to wait for") + return + } + + pendingActions = [...actions] + console.log(`Waiting for ${pendingActions.join(", ")}`) + while (now <= deadline) { + const checkRuns = await github.paginate(github.rest.checks.listForRef, { + owner: 'cloudquery', + repo: context.repo.repo, + ref: context.payload.pull_request.head.sha, + status: 'completed', + per_page: 100 + }) + const runsWithPossibleDuplicates = checkRuns.map(({id, name, conclusion}) => ({id, name, conclusion})) + const runs = runsWithPossibleDuplicates.filter((run, index, self) => self.findIndex(({id}) => id === run.id) === index) + console.log(`Got the following check runs: ${JSON.stringify(runs)}`) + const matchingRuns = runs.filter(({name}) => actions.includes(name)) + const failedRuns = matchingRuns.filter(({conclusion}) => conclusion !== 'success') + if (failedRuns.length > 0) { + throw new Error(`The following required workflows failed: ${failedRuns.map(({name}) => name).join(", ")}`) + } + console.log(`Matching runs: ${matchingRuns.map(({name}) => name).join(", ")}`) + console.log(`Actions: ${actions.join(", ")}`) + if (matchingRuns.length === actions.length) { + console.log("All required workflows have passed") + return + } + pendingActions = actions.filter(action => !runs.some(({name}) => name === action)) + console.log(`Waiting for ${pendingActions.join(", ")}`) + await new Promise(r => setTimeout(r, 60000)); + now = new Date().getTime() + } + throw new Error(`Timed out waiting for ${pendingActions.join(', ')}`) +}