From ab001a8357bd17965c5f7ca483c1eddd7687420e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 14:15:24 +0100 Subject: [PATCH 1/2] simplify matrix generation step of build --- .github/workflows/_extension_distribution.yml | 47 ++----------------- 1 file changed, 5 insertions(+), 42 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 021f744..9d5001b 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -49,7 +49,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" # Enable building the DuckDB Shell build_duckdb_shell: required: false @@ -115,22 +115,6 @@ jobs: osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }} wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }} steps: - - uses: actions/checkout@v4 - name: Checkout override repository - if: ${{inputs.override_repository != ''}} - with: - repository: ${{ inputs.override_repository }} - ref: ${{ inputs.override_ref }} - fetch-depth: 0 - submodules: 'true' - - - uses: actions/checkout@v4 - name: Checkout current repository - if: ${{inputs.override_repository == ''}} - with: - fetch-depth: 0 - submodules: 'true' - - uses: actions/checkout@v4 name: Checkout Extension CI tools with: @@ -138,34 +122,13 @@ jobs: ref: ${{ inputs.ci_tools_version }} repository: ${{ inputs.override_ci_tools_repository }} - - name: Checkout DuckDB to version - if: ${{inputs.duckdb_version != ''}} - run: | - DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version - - - name: Tag extension - if: ${{inputs.extension_tag != ''}} - run: | - git tag ${{ inputs.extension_tag }} - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != '' && (inputs.duckdb_version == 'main' || inputs.duckdb_version[0] == 'v') }} - run: | - echo "When duckdb_tag is provied an explcit git ref is expected" && exit 1 - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != ''}} - run: | - DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag - - id: parse-matrices run: | mkdir build - make output_distribution_matrix | tail -n +2 > build/distribution_matrix.json - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - id: set-matrix-linux run: | From 74cca858bd3a654e4ae2595e6b42cd14d65b9787 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 14:30:33 +0100 Subject: [PATCH 2/2] simplify matrix creation step for deploy --- .github/workflows/_extension_deploy.yml | 25 ++++++++++++------- .github/workflows/_extension_distribution.yml | 1 + 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index 59be0ef..d9d68bc 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -18,6 +18,16 @@ on: duckdb_version: required: true type: string + # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`. + # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264 + ci_tools_version: + required: true + type: string + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "duckdb/extension-ci-tools" # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' exclude_archs: required: false @@ -47,7 +57,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" jobs: generate_matrix: @@ -57,18 +67,15 @@ jobs: deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} steps: - uses: actions/checkout@v4 + name: Checkout Extension CI tools with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} - id: parse-matrices run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty deploy_matrix="`cat deploy_matrix.json`" echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 9d5001b..bcc4c6d 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -105,6 +105,7 @@ on: required: false type: boolean default: false + jobs: generate_matrix: name: Generate matrix