Merge branch 'main' into more_jemalloc

lnkuiper · Mar 15, 2024 · ea24186 · ea24186
2 parents c8f6f0b + 534f3c8
commit ea24186
Show file tree

Hide file tree

Showing 84 changed files with 1,334 additions and 532 deletions.
diff --git a/.github/actions/build_extensions/action.yml b/.github/actions/build_extensions/action.yml
@@ -96,7 +96,7 @@ runs:
       if: inputs.vcpkg_build == 1
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
     - name: Set vcpkg env variables
       if: inputs.vcpkg_build == 1

diff --git a/.github/actions/manylinux_2014_setup/action.yml b/.github/actions/manylinux_2014_setup/action.yml
@@ -96,7 +96,7 @@ runs:
       if: ${{ inputs.vcpkg == 1 }}
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
     - name: Install OpenSSL
       if: ${{ inputs.openssl == 1 }}

diff --git a/.github/actions/ubuntu_18_setup/action.yml b/.github/actions/ubuntu_18_setup/action.yml
@@ -84,7 +84,7 @@ runs:
       if: ${{ inputs.vcpkg == 1 }}
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
 
     - name: Setup Ccache
       if: ${{ inputs.ccache == 1 }}

diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv
@@ -13,5 +13,5 @@ substrait,https://github.com/duckdb/substrait,e75ba6a9a43d75d8c72ecc38e6f751fd8e
 arrow,https://github.com/duckdb/arrow,9e10240da11f61ea7fbfe3fc9988ffe672ccd40f,no-windows
 aws,https://github.com/duckdb/duckdb_aws,348ae2625de86ab760f80a43eb76e4441cd01354,
 azure,https://github.com/duckdb/duckdb_azure,1fe568d3eb3c8842118e395ba8031e2a8566daed,
-spatial,https://github.com/duckdb/duckdb_spatial,b9cd1904a2d243b9937294ae94d4c38e7ad289f4,
+spatial,https://github.com/duckdb/duckdb_spatial,05c4ba01c500140287bf6946fb6910122e5c2acf,
 iceberg,https://github.com/duckdb/duckdb_iceberg,51ba9564859698c29db4165f17143a2f6af2bb18,
diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake
@@ -19,7 +19,7 @@ if (NOT MINGW)
     duckdb_extension_load(aws
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_aws
-            GIT_TAG 043890c2ad83c4a1375417e2d5d085a33f79a217
+            GIT_TAG f7b8729f1cce5ada5d4add70e1486de50763fb97
             )
 endif()
 
@@ -28,7 +28,7 @@ if (NOT MINGW)
     duckdb_extension_load(azure
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_azure
-            GIT_TAG 3ad0348334d2a263e5f19ef08fd02311bdae82f3
+            GIT_TAG 9e3e5b8addab5939b4ca4ac5f8997218a7228b8f
             )
 endif()
 
@@ -65,7 +65,7 @@ if (NOT MINGW)
     duckdb_extension_load(spatial
             DONT_LINK LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_spatial.git
-            GIT_TAG b9cd1904a2d243b9937294ae94d4c38e7ad289f4
+            GIT_TAG 05c4ba01c500140287bf6946fb6910122e5c2acf
             INCLUDE_DIR spatial/include
             TEST_DIR test/sql
             )

diff --git a/.github/patches/extensions/postgres_scanner/rebind.patch b/.github/patches/extensions/postgres_scanner/rebind.patch
@@ -10,6 +10,31 @@ index 396664d..eb8f621 100644
  };
 
  class PostgresQueryFunction : public TableFunction {
+diff --git a/src/include/storage/postgres_catalog.hpp b/src/include/storage/postgres_catalog.hpp
+index 7e90c18..1b887fa 100644
+--- a/src/include/storage/postgres_catalog.hpp
++++ b/src/include/storage/postgres_catalog.hpp
+@@ -72,6 +72,20 @@ public:
+
+ 	void ClearCache();
+
++	//! Whether or not this catalog should search a specific type with the standard priority
++	CatalogLookupBehavior CatalogTypeLookupRule(CatalogType type) const override {
++		switch (type) {
++		case CatalogType::INDEX_ENTRY:
++		case CatalogType::TABLE_ENTRY:
++		case CatalogType::TYPE_ENTRY:
++		case CatalogType::VIEW_ENTRY:
++			return CatalogLookupBehavior::STANDARD;
++		default:
++			// unsupported type (e.g. scalar functions, aggregates, ...)
++			return CatalogLookupBehavior::NEVER_LOOKUP;
++		}
++	}
++
+ private:
+ 	void DropSchema(ClientContext &context, DropInfo &info) override;
+
 diff --git a/src/postgres_extension.cpp b/src/postgres_extension.cpp
 index 81bd46e..6e18d5a 100644
 --- a/src/postgres_extension.cpp

diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv
diff --git a/.github/workflows/InvokeCI.yml b/.github/workflows/InvokeCI.yml
@@ -0,0 +1,51 @@
+name: InvokeCI
+on:
+  workflow_dispatch:
+    inputs:
+      override_git_describe:
+        type: string
+      git_ref:
+        type: string
+      skip_tests:
+        type: string
+
+jobs:
+  osx:
+    uses: ./.github/workflows/OSX.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
+
+  linux-release:
+    uses: ./.github/workflows/LinuxRelease.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
+
+  windows:
+    uses: ./.github/workflows/Windows.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
+
+  python:
+    uses: ./.github/workflows/Python.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
+
+  java:
+    uses: ./.github/workflows/Java.yml
+    secrets: inherit
+    with:
+      override_git_describe: ${{ inputs.override_git_describe }}
+      git_ref: ${{ inputs.git_ref }}
+      skip_tests: ${{ inputs.skip_tests }}
diff --git a/.github/workflows/Java.yml b/.github/workflows/Java.yml
@@ -1,6 +1,21 @@
 name: Java JDBC
 on:
+  workflow_call:
+    inputs:
+      override_git_describe:
+        type: string
+      git_ref:
+        type: string
+      skip_tests:
+        type: string
   workflow_dispatch:
+    inputs:
+      override_git_describe:
+        type: string
+      git_ref:
+        type: string
+      skip_tests:
+        type: string
   repository_dispatch:
   push:
     branches:
@@ -32,12 +47,13 @@ on:
       - '!.github/workflows/Java.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{
+  group: java-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{
     github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha
-    }}
+    }}-${{ inputs.override_git_describe }}
   cancel-in-progress: true
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
 
 jobs:
   java-linux-amd64:
@@ -57,28 +73,34 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
 
       - uses: ./.github/actions/manylinux_2014_setup
         with:
           ninja-build: 1
           ccache: 1
           jdk: 1
+          python_alias: 1
+          aws-cli: 1
 
       - name: Build
         shell: bash
         run: make
 
       - name: Java Tests
         shell: bash
-        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ inputs.skip_tests != 'true' }}
         working-directory: tools/jdbc
         run: make test_release
 
       - name: Deploy
         shell: bash
-        run: >
-          python3.8 scripts/asset-upload-gha.py
-          duckdb_jdbc-linux-amd64.jar=build/release/tools/jdbc/duckdb_jdbc.jar
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          cp build/release/tools/jdbc/duckdb_jdbc.jar duckdb_jdbc-linux-amd64.jar
+          ./scripts/upload-assets-to-staging.sh duckdb_jdbc-linux-amd64.jar
       - uses: actions/upload-artifact@v3
         with:
           name: java-linux-amd64
@@ -102,6 +124,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
 
       - uses: ./.github/actions/ubuntu_18_setup
         with:
@@ -119,9 +142,12 @@ jobs:
 
       - name: Deploy
         shell: bash
-        run: >
-          python3.8 scripts/asset-upload-gha.py
-          duckdb_jdbc-linux-aarch64.jar=build/release/tools/jdbc/duckdb_jdbc.jar
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          cp build/release/tools/jdbc/duckdb_jdbc.jar duckdb_jdbc-linux-aarch64.jar
+          ./scripts/upload-assets-to-staging.sh duckdb_jdbc-linux-aarch64.jar
 
       - uses: actions/upload-artifact@v3
         with:
@@ -138,6 +164,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
@@ -156,15 +183,18 @@ jobs:
 
           cmake --build . --config Release
       - name: Java Tests
-        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ inputs.skip_tests != 'true' }}
         shell: bash
         working-directory: tools/jdbc
         run: make test_release
       - name: Deploy
         shell: bash
-        run: >
-          python scripts/asset-upload-gha.py
-          duckdb_jdbc-windows-amd64.jar=tools/jdbc/duckdb_jdbc.jar
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          cp tools/jdbc/duckdb_jdbc.jar duckdb_jdbc-windows-amd64.jar
+          ./scripts/upload-assets-to-staging.sh duckdb_jdbc-windows-amd64.jar
       - uses: actions/upload-artifact@v3
         with:
           name: java-windows-amd64
@@ -186,6 +216,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
@@ -198,7 +229,7 @@ jobs:
         shell: bash
         run: make
       - name: Java Tests
-        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ inputs.skip_tests != 'true' }}
         shell: bash
         working-directory: tools/jdbc
         run: make test_release
@@ -208,9 +239,12 @@ jobs:
           (cd examples/jdbc; make; make maven)
       - name: Deploy
         shell: bash
-        run: >
-          python scripts/asset-upload-gha.py
-          duckdb_jdbc-osx-universal.jar=build/release/tools/jdbc/duckdb_jdbc.jar
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          cp build/release/tools/jdbc/duckdb_jdbc.jar duckdb_jdbc-osx-universal.jar
+          ./scripts/upload-assets-to-staging.sh duckdb_jdbc-osx-universal.jar
       - uses: actions/upload-artifact@v3
         with:
           name: java-osx-universal
@@ -231,6 +265,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
 
       - shell: bash
         run: mkdir jdbc-artifacts
@@ -289,7 +324,7 @@ jobs:
   jdbc-compliance:
     name: JDBC Compliance
     runs-on: ubuntu-20.04
-    if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+    if: ${{ inputs.skip_tests != 'true' }}
     needs: java-linux-amd64
     container: quay.io/pypa/manylinux2014_x86_64
     env:
@@ -300,12 +335,15 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
 
       - uses: ./.github/actions/manylinux_2014_setup
         with:
           ninja-build: 1
           ccache: 1
           jdk: 1
+          python_alias: 1
+          aws-cli: 1
 
       - name: Install
         shell: bash