-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Add wait for all workflow (#505)
- Loading branch information
Showing
14 changed files
with
248 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| name: "Test AWS Encryption Policies" | ||
|
|
||
| on: | ||
| pull_request: | ||
| paths: | ||
| - "transformations/aws/encryption/**" | ||
| - ".github/workflows/transformations_aws_encryption.yml" | ||
| - "transformations/aws/macros/**" | ||
| - "transformations/aws/models/**" | ||
| - "transformations/macros/**" | ||
| - ".github/workflows/wait_for_required_workflows.yml" | ||
| - "scripts/workflows/wait_for_required_workflows.js" | ||
| push: | ||
| branches: | ||
| - main | ||
| paths: | ||
| - "transformations/aws/encryption/**" | ||
| - ".github/workflows/transformations_aws_encryption.yml" | ||
| - "transformations/aws/macros/**" | ||
| - "transformations/aws/models/**" | ||
| - "transformations/macros/**" | ||
|
|
||
| env: | ||
| SNOW_USER: ${{ secrets.SNOW_USER }} | ||
| SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} | ||
| # DBT assumes the account is in the form of <account>.<region> | ||
| SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" | ||
| SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} | ||
| SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} | ||
| SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} | ||
| SNOW_REGION: ${{ secrets.SNOW_REGION }} | ||
|
|
||
| jobs: | ||
| prepare: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} | ||
| postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} | ||
| snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} | ||
| bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - uses: actions/github-script@v7 | ||
| id: set-result | ||
| env: | ||
| TRANSFORMATION_DIR: transformations/aws/encryption | ||
| with: | ||
| script: | | ||
| const fs = require('fs/promises'); | ||
| const { TRANSFORMATION_DIR: transformation_dir } = process.env; | ||
| const [postgres, snowflake, bigquery] = await Promise.all([ | ||
| fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), | ||
| fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), | ||
| fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), | ||
| ]); | ||
| return { | ||
| transformation_dir, | ||
| postgres, | ||
| snowflake, | ||
| bigquery, | ||
| }; | ||
| transformations-aws-encryption: | ||
| permissions: | ||
| id-token: 'write' | ||
| contents: 'read' | ||
| name: ${{ needs.prepare.outputs.transformation_dir }} | ||
| needs: prepare | ||
| timeout-minutes: 30 | ||
| runs-on: ubuntu-latest | ||
| defaults: | ||
| run: | ||
| working-directory: ${{ needs.prepare.outputs.transformation_dir }} | ||
| services: | ||
| postgres: | ||
| image: postgres:11 | ||
| env: | ||
| POSTGRES_PASSWORD: pass | ||
| POSTGRES_USER: postgres | ||
| POSTGRES_DB: postgres | ||
| ports: | ||
| - 5432:5432 | ||
| # Set health checks to wait until postgres has started | ||
| options: >- | ||
| --health-cmd pg_isready | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Authenticate to Google Cloud | ||
| uses: 'google-github-actions/auth@v2' | ||
| if: needs.prepare.outputs.bigquery == 'true' | ||
| with: | ||
| workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' | ||
| service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: "3.9" | ||
| cache: "pip" | ||
| cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" | ||
| - name: Install dependencies | ||
| run: pip install -r requirements.txt | ||
| - name: Setup CloudQuery | ||
| uses: cloudquery/setup-cloudquery@v3 | ||
| with: | ||
| version: v5.0.1 | ||
| - name: Test Postgres | ||
| run: | | ||
| cloudquery migrate tests/postgres.yml | ||
| dbt seed --target dev-pg --profiles-dir ./tests | ||
| dbt run --target dev-pg --profiles-dir ./tests | ||
| if: needs.prepare.outputs.postgres == 'true' | ||
| env: | ||
| CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres | ||
| - name: Test Snowflake | ||
| run: | | ||
| cloudquery migrate tests/snowflake.yml | ||
| dbt seed --target dev-pg --profiles-dir ./tests | ||
| dbt run --target dev-snowflake --profiles-dir ./tests | ||
| if: needs.prepare.outputs.snowflake == 'true' | ||
| env: | ||
| SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" | ||
| - name: Test BigQuery | ||
| if: needs.prepare.outputs.bigquery == 'true' | ||
| run: | | ||
| cloudquery migrate tests/bigquery.yml | ||
| dbt seed --target dev-pg --profiles-dir ./tests | ||
| dbt run --target dev-bigquery --profiles-dir ./tests |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| name: Wait for all required workflows to pass | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| on: | ||
| pull_request: | ||
| branches: | ||
| - main | ||
|
|
||
| jobs: | ||
| wait_for_required_workflows: | ||
| timeout-minutes: 60 | ||
| name: wait-for-required-workflows | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Get changed files | ||
| id: changed-files | ||
| uses: tj-actions/changed-files@v41 | ||
| - uses: actions/github-script@v7 | ||
| env: | ||
| FILES: ${{ steps.changed-files.outputs.all_modified_files }} | ||
| with: | ||
| script: | | ||
| const script = require('./scripts/workflows/wait_for_required_workflows.js') | ||
| await script({github, context}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| module.exports = async ({github, context}) => { | ||
| const files = process.env.FILES.split(' ') | ||
| console.log(files) | ||
| if (files.length >= 300) { | ||
| // This is a GitHub limitation https://github.com/cloudquery/cloudquery/issues/2688 | ||
| throw new Error('Too many files changed. Skipping check. Please split your PR into multiple ones.') | ||
| } | ||
|
|
||
| const path = require("path"); | ||
| const fs = require("fs"); | ||
| let now = new Date().getTime() | ||
| const deadline = now + 60 * 1000 * 50 | ||
| const matchesWorkflow = (file, action) => { | ||
| if (!file.startsWith('.github/workflows/')) { | ||
| return false | ||
| } | ||
| try { | ||
| const contents = fs.readFileSync(file, 'utf8'); | ||
| return contents.includes(`"${action}`) | ||
| } catch { | ||
| return false | ||
| } | ||
| } | ||
| const matchesFile = (action) => { | ||
| return files.some(file => file.startsWith(`${action}/`) || matchesWorkflow(file, action)) | ||
| } | ||
|
|
||
| const testAll = files.includes("scripts/workflows/wait_for_required_workflows.js") || files.includes(".github/workflows/wait_for_required_workflows.yml") | ||
| const transformations = fs.readdirSync("transformations", {withFileTypes: true, recursive: true}).filter(dirent => dirent.isFile() && dirent.name === "dbt_project.yml").map(dirent => dirent.path) | ||
| let actions = transformations.filter(action => testAll || matchesFile(action)) | ||
| if (actions.length === 0) { | ||
| console.log("No actions to wait for") | ||
| return | ||
| } | ||
|
|
||
| pendingActions = [...actions] | ||
| console.log(`Waiting for ${pendingActions.join(", ")}`) | ||
| while (now <= deadline) { | ||
| const checkRuns = await github.paginate(github.rest.checks.listForRef, { | ||
| owner: 'cloudquery', | ||
| repo: context.repo.repo, | ||
| ref: context.payload.pull_request.head.sha, | ||
| status: 'completed', | ||
| per_page: 100 | ||
| }) | ||
| const runsWithPossibleDuplicates = checkRuns.map(({id, name, conclusion}) => ({id, name, conclusion})) | ||
| const runs = runsWithPossibleDuplicates.filter((run, index, self) => self.findIndex(({id}) => id === run.id) === index) | ||
| console.log(`Got the following check runs: ${JSON.stringify(runs)}`) | ||
| const matchingRuns = runs.filter(({name}) => actions.includes(name)) | ||
| const failedRuns = matchingRuns.filter(({conclusion}) => conclusion !== 'success') | ||
| if (failedRuns.length > 0) { | ||
| throw new Error(`The following required workflows failed: ${failedRuns.map(({name}) => name).join(", ")}`) | ||
| } | ||
| console.log(`Matching runs: ${matchingRuns.map(({name}) => name).join(", ")}`) | ||
| console.log(`Actions: ${actions.join(", ")}`) | ||
| if (matchingRuns.length === actions.length) { | ||
| console.log("All required workflows have passed") | ||
| return | ||
| } | ||
| pendingActions = actions.filter(action => !runs.some(({name}) => name === action)) | ||
| console.log(`Waiting for ${pendingActions.join(", ")}`) | ||
| await new Promise(r => setTimeout(r, 60000)); | ||
| now = new Date().getTime() | ||
| } | ||
| throw new Error(`Timed out waiting for ${pendingActions.join(', ')}`) | ||
| } |