Merge branch 'feature' into left_projection_map

lnkuiper · Oct 28, 2024 · c203460 · c203460
2 parents b3a2ed4 + b83a0be
commit c203460
Show file tree

Hide file tree

Showing 214 changed files with 5,911 additions and 4,028 deletions.
diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv
@@ -15,4 +15,4 @@ aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb9
 azure,https://github.com/duckdb/duckdb_azure,09623777a366572bfb8fa53e47acdf72133a360e,
 spatial,https://github.com/duckdb/duckdb_spatial,bb9c829693965f029eb5a312aefed4c538fad781,
 iceberg,https://github.com/duckdb/duckdb_iceberg,d89423c2ff90a0b98a093a133c8dfe2a55b9e092,
-vss,https://github.com/duckdb/duckdb_vss,9ff608f1edc6a1da2e41ee41b1ef42ba3169e71c,
+vss,https://github.com/duckdb/duckdb_vss,74137d802e0867966a604ba7dc49eefc18d1ee7f,
diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake
@@ -149,7 +149,7 @@ duckdb_extension_load(vss
         LOAD_TESTS
         DONT_LINK
         GIT_URL https://github.com/duckdb/duckdb_vss
-        GIT_TAG dd880d6121c0f3dff27131e54e057c9db0f1c710
+        GIT_TAG 74137d802e0867966a604ba7dc49eefc18d1ee7f
         TEST_DIR test/sql
         APPLY_PATCHES
     )

diff --git a/.github/workflows/ExtendedTests.yml b/.github/workflows/ExtendedTests.yml
@@ -32,9 +32,7 @@ jobs:
        CXX: clang++
        GEN: ninja
        BUILD_BENCHMARK: 1
-       BUILD_TPCH: 1
-       BUILD_TPCDS: 1
-       BUILD_HTTPFS: 1
+       CORE_EXTENSIONS: "tpch;tpcds;httpfs"
 
      steps:
      - uses: actions/checkout@v4
@@ -114,9 +112,7 @@ jobs:
        CXX: clang++
        GEN: ninja
        BUILD_BENCHMARK: 1
-       BUILD_TPCH: 1
-       BUILD_TPCDS: 1
-       BUILD_HTTPFS: 1
+       CORE_EXTENSIONS: "tpch;tpcds;httpfs"
 
      steps:
      - uses: actions/checkout@v4
@@ -195,10 +191,8 @@ jobs:
        CXX: g++
        GEN: ninja
        BUILD_BENCHMARK: 1
-       BUILD_TPCH: 1
-       BUILD_TPCDS: 1
-       BUILD_HTTPFS: 1
        BUILD_JEMALLOC: 1
+       CORE_EXTENSIONS: "tpch;tpcds;httpfs"
 
      steps:
        - uses: actions/checkout@v4
@@ -282,10 +276,8 @@ jobs:
        CXX: g++
        GEN: ninja
        BUILD_BENCHMARK: 1
-       BUILD_TPCH: 1
-       BUILD_TPCDS: 1
-       BUILD_HTTPFS: 1
        BUILD_JEMALLOC: 1
+       CORE_EXTENSIONS: "tpch;tpcds;httpfs"
 
      steps:
        - uses: actions/checkout@v4

diff --git a/.github/workflows/ExtraTests.yml b/.github/workflows/ExtraTests.yml
@@ -18,10 +18,8 @@ jobs:
     CXX: g++-10
     GEN: ninja
     BUILD_BENCHMARK: 1
-    BUILD_TPCH: 1
-    BUILD_TPCDS: 1
-    BUILD_HTTPFS: 1
     BUILD_JEMALLOC: 1
+    CORE_EXTENSIONS: "tpcd;tpcds;httpfs"
 
   steps:
     - uses: actions/checkout@v4

diff --git a/.github/workflows/InternalIssuesCreateMirror.yml b/.github/workflows/InternalIssuesCreateMirror.yml
@@ -56,11 +56,11 @@ jobs:
       - name: Set label environment variable
         run: |
           if ${{ github.event.label.name == 'reproduced' }}; then
-            echo "LABEL=needs label" >> $GITHUB_ENV
-            echo "UNLABEL=needs triage" >> $GITHUB_ENV
+            echo "LABEL=reproduced" >> $GITHUB_ENV
+            echo "UNLABEL=under review" >> $GITHUB_ENV
           else
-            echo "LABEL=needs triage" >> $GITHUB_ENV
-            echo "UNLABEL=needs label" >> $GITHUB_ENV
+            echo "LABEL=under review" >> $GITHUB_ENV
+            echo "UNLABEL=reproduced" >> $GITHUB_ENV
           fi
 
       - name: Create or label issue

diff --git a/.github/workflows/Julia.yml b/.github/workflows/Julia.yml
@@ -93,7 +93,7 @@ jobs:
       - name: Build DuckDB
         shell: bash
         run: |
-            BUILD_TPCH=1 BUILD_ICU=1 BUILD_JEMALLOC=1 make
+            CORE_EXTENSIONS="tpch;icu" BUILD_JEMALLOC=1 make
 
       - name: Run Tests
         shell: bash

diff --git a/.github/workflows/LinuxRelease.yml b/.github/workflows/LinuxRelease.yml
@@ -448,4 +448,4 @@ jobs:
         shell: bash
         run: |
           mkdir -p build/release
-          (cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DBUILD_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .)
+          (cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DCORE_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .)
diff --git a/.github/workflows/Main.yml b/.github/workflows/Main.yml
@@ -94,14 +94,8 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
       BUILD_JEMALLOC: 1
-      BUILD_EXTENSIONS: ""
+      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json"
       RUN_SLOW_VERIFIERS: 1
 
     steps:
@@ -139,13 +133,8 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
       BUILD_JEMALLOC: 1
+      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json"
 
     steps:
     - uses: actions/checkout@v4

diff --git a/.github/workflows/NightlyTests.yml b/.github/workflows/NightlyTests.yml
@@ -71,13 +71,8 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
       BUILD_JEMALLOC: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
       DISABLE_SANITIZER: 1
       CRASH_ON_ASSERT: 1
       RUN_SLOW_VERIFIERS: 1
@@ -114,12 +109,9 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_JSON: 1
-      SMALLER_BINARY: 1
       BUILD_JEMALLOC: 1
+      CORE_EXTENSIONS: "icu;tpch;tpcds;json"
+      SMALLER_BINARY: 1
     steps:
     - uses: actions/checkout@v3
       with:
@@ -158,7 +150,7 @@ jobs:
       CXX: /home/runner/work/llvm/bin/clang++
       LD: /home/runner/work/llvm/bin/ld.lld
       EXTENSION_STATIC_BUILD: 1
-      BUILD_JSON: 1
+      CORE_EXTENSIONS: "json"
       TREAT_WARNINGS_AS_ERRORS: 1
 
     steps:
@@ -242,7 +234,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p build/release
-          (cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DBUILD_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .)
+          (cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DCORE_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .)
 
       - name: Test
         shell: bash
@@ -477,7 +469,7 @@ jobs:
 
       - name: Build
         shell: bash
-        run: BUILD_TPCH=1 make
+        run: CORE_EXTENSIONS="tpch" make
 
       - name: Start Minio
         shell: bash
@@ -501,13 +493,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
       DISABLE_STRING_INLINE: 1
       DESTROY_UNPINNED_BLOCKS: 1
       ALTERNATIVE_VERIFY: 1
@@ -544,10 +530,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_JSON: 1
+      CORE_EXTENSIONS: "parquet;json;tpch;tpcds"
       LATEST_STORAGE: 1
       BLOCK_VERIFICATION: 1
 
@@ -597,13 +580,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
       VERIFY_VECTOR: ${{ matrix.vector_type }}
 
     steps:
@@ -637,13 +614,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_PARQUET: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "icu;parquet;tpch;tpcds;fts;json;inet"
       FORCE_ASYNC_SINK_SOURCE: 1
 
     steps:
@@ -678,10 +649,8 @@ jobs:
      CXX: g++-10
      GEN: ninja
      BUILD_BENCHMARK: 1
-     BUILD_TPCH: 1
-     BUILD_TPCDS: 1
-     BUILD_HTTPFS: 1
      BUILD_JEMALLOC: 1
+     CORE_EXTENSIONS: "tpch;tpcds;httpfs"
 
    steps:
      - name: Checkout
@@ -761,13 +730,8 @@ jobs:
       CC: clang
       CXX: clang++
       GEN: ninja
-      BUILD_ICU: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_FTS: 1
-      BUILD_JSON: 1
       BUILD_JEMALLOC: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "icu;tpch;tpcds;fts;json;inet"
       TSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-thread-suppressions.txt
 
     steps:
@@ -844,8 +808,7 @@ jobs:
       CXX: g++-10
       GEN: ninja
       BLOCK_ALLOC_SIZE: 16384
-      BUILD_JSON: 1
-      BUILD_PARQUET: 1
+      CORE_EXTENSIONS: "json;parquet"
 
     steps:
       - uses: actions/checkout@v3
@@ -1007,11 +970,7 @@ jobs:
     name: Linux HTTPFS
     runs-on: ubuntu-20.04
     env:
-      BUILD_HTTPFS: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_PARQUET: 1
-      BUILD_JSON: 1
+      CORE_EXTENSIONS: "json;parquet;tpch;tpcds;httpfs"
       S3_TEST_SERVER_AVAILABLE: 1
       AWS_DEFAULT_REGION: eu-west-1
       AWS_ACCESS_KEY_ID: minio_duckdb_user

diff --git a/.github/workflows/Python.yml b/.github/workflows/Python.yml
@@ -52,7 +52,7 @@ concurrency:
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
   OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
-  CIBW_TEST_SKIP: ${{ inputs.skip_tests == 'true' && '*-*' || '' }}
+  CIBW_TEST_SKIP: ${{ inputs.skip_tests == 'true' && '*-*' || 'cp37-*' }}
 
 jobs:
 #  This is just a sanity check of Python 3.9 running with Arrow
@@ -62,9 +62,10 @@ jobs:
 
     env:
       CIBW_BUILD: 'cp39-manylinux_x86_64'
-      CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose'
+      CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose && USE_ACTUAL_SPARK=true JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) SPARK_HOME={project}/spark_installation/spark python -m pytest {project}/tests/fast/spark --verbose'
       PYTEST_TIMEOUT: '600'
       CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}'
+      CIBW_BEFORE_ALL: 'cd {project} && ./scripts/install_spark_in_cibuildwheels_linux_container.sh'
 
     steps:
     - uses: actions/checkout@v3

diff --git a/.github/workflows/Regression.yml b/.github/workflows/Regression.yml
@@ -43,12 +43,8 @@ jobs:
       CXX: g++-10
       GEN: ninja
       BUILD_BENCHMARK: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_HTTPFS: 1
       BUILD_JEMALLOC: 1
-      BUILD_JSON: 1
-      CORE_EXTENSIONS: "inet"
+      CORE_EXTENSIONS: "json;tpch;tpcds;httpfs;inet"
 
     steps:
       - uses: actions/checkout@v4
@@ -148,8 +144,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
+      CORE_EXTENSIONS: "tpch;tpcds"
 
     steps:
       - uses: actions/checkout@v4
@@ -229,10 +224,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_JSON: 1
-      BUILD_PARQUET: 1
+      CORE_EXTENSIONS: "tpch;tpcds;json;parquet"
       EXTENSION_STATIC_BUILD: 1
     steps:
       - uses: actions/checkout@v4
@@ -274,8 +266,7 @@ jobs:
       CC: gcc-10
       CXX: g++-10
       GEN: ninja
-      BUILD_TPCH: 1
-      BUILD_HTTPFS: 1
+      CORE_EXTENSIONS: "tpch;httpfs"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml
@@ -239,7 +239,7 @@ jobs:
        - name: Build
          shell: msys2 {0}
          run: |
-           cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DBUILD_EXTENSIONS=parquet -DOVERRIDE_GIT_DESCRIBE="$OVERRIDE_GIT_DESCRIBE"
+           cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DCORE_EXTENSIONS='parquet' -DOVERRIDE_GIT_DESCRIBE="$OVERRIDE_GIT_DESCRIBE"
            cmake --build . --config Release 
 
        - name: Test

diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -40,15 +40,8 @@ jobs:
       run: cov-analysis-linux64/bin/cov-build --dir cov-int make
       env:
         PYTHON_USER_SPACE: 1
-        BUILD_PYTHON: 1
-        BUILD_AUTOCOMPLETE: 1
-        BUILD_ICU: 1
-        BUILD_TPCDS: 1
-        BUILD_TCPE: 1
-        BUILD_FTS: 1
-        BUILD_HTTPFS: 1
-        BUILD_JSON: 1
-        CORE_EXTENSIONS: "inet"
+        BUILD_TPCE: 1
+        CORE_EXTENSIONS: "autocomplete;icu;tpcds;tpch;fts;httpfs;json;inet"
 
     - name: Upload the result
       run: |

diff --git a/data/csv/header_bug.csv b/data/csv/header_bug.csv
@@ -0,0 +1,4 @@
+a,b
+c,d
+e,f
+g,h