Merge pull request #103 from samansmink/refactor-matrix-generation

Refactor matrix generation
duckdb · Nov 12, 2024 · 0315837 · 0315837
2 parents eb07f23 + 74cca85
commit 0315837
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 50 deletions.
diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml
@@ -18,6 +18,16 @@ on:
       duckdb_version:
         required: true
         type: string
+      # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`.
+      # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264
+      ci_tools_version:
+        required: true
+        type: string
+      # Override the repo for the CI tools (for testing CI tools itself)
+      override_ci_tools_repository:
+        required: false
+        type: string
+        default: "duckdb/extension-ci-tools"
       # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64'
       exclude_archs:
         required: false
@@ -47,7 +57,7 @@ on:
       matrix_parse_script:
         required: false
         type: string
-        default: "./duckdb/scripts/modify_distribution_matrix.py"
+        default: "./extension-ci-tools/scripts/modify_distribution_matrix.py"
 
 jobs:
   generate_matrix:
@@ -57,18 +67,15 @@ jobs:
       deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }}
     steps:
       - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
         with:
-          fetch-depth: 0
-          submodules: 'true'
-
-      - name: Checkout DuckDB to version
-        run: |
-          cd duckdb
-          git checkout ${{ inputs.duckdb_version }}
+          path: 'extension-ci-tools'
+          ref: ${{ inputs.ci_tools_version }}
+          repository: ${{ inputs.override_ci_tools_repository }}
 
       - id: parse-matrices
         run: |
-          python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
           deploy_matrix="`cat deploy_matrix.json`"
           echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT
           echo `cat $GITHUB_OUTPUT`

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
@@ -100,6 +100,7 @@ on:
         required: false
         type: boolean
         default: false
+
 jobs:
   generate_matrix:
     name: Generate matrix
@@ -110,57 +111,20 @@ jobs:
       osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }}
       wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }}
     steps:
-      - uses: actions/checkout@v4
-        name: Checkout override repository
-        if: ${{inputs.override_repository != ''}}
-        with:
-          repository: ${{ inputs.override_repository }}
-          ref: ${{ inputs.override_ref }}
-          fetch-depth: 0
-          submodules: 'true'
-
-      - uses: actions/checkout@v4
-        name: Checkout current repository
-        if: ${{inputs.override_repository == ''}}
-        with:
-          fetch-depth: 0
-          submodules: 'true'
-
       - uses: actions/checkout@v4
         name: Checkout Extension CI tools
         with:
           path: 'extension-ci-tools'
           ref: ${{ inputs.ci_tools_version }}
           repository: ${{ inputs.override_ci_tools_repository }}
 
-      - name: Checkout DuckDB to version
-        if: ${{inputs.duckdb_version != ''}}
-        run: |
-          DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version
-
-      - name: Tag extension
-        if: ${{inputs.extension_tag != ''}}
-        run: |
-          git tag ${{ inputs.extension_tag }}
-
-      - name: Tag DuckDB extension
-        if: ${{inputs.duckdb_tag != '' && (inputs.duckdb_version == 'main' || inputs.duckdb_version[0] == 'v') }}
-        run: |
-          echo "When duckdb_tag is provied an explcit git ref is expected" && exit 1
-
-      - name: Tag DuckDB extension
-        if: ${{inputs.duckdb_tag != ''}}
-        run: |
-          DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag
-
       - id: parse-matrices
         run: |
           mkdir build
-          make output_distribution_matrix | tail -n +2 > build/distribution_matrix.json
-          python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
-          python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
-          python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
-          python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
+          python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
 
       - id: set-matrix-linux
         run: |