Merge branch 'main' into aggregate-ec

Author: jeewonhh

.github/actions/build_extensions/action.yml

@@ -99,7 +99,7 @@ runs:
       if: inputs.vcpkg_build == 1
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+        vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
     - name: Set vcpkg env variables
       if: inputs.vcpkg_build == 1

.github/actions/build_extensions_dockerized/action.yml

@@ -114,7 +114,7 @@ runs:
         run: |
           docker build \
             --build-arg 'vcpkg_url=https://github.com/microsoft/vcpkg.git' \
-            --build-arg 'vcpkg_commit=a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6' \
+            --build-arg 'vcpkg_commit=5e5d0e1cd7785623065e77eff011afdeec1a3574' \
             --build-arg 'extra_toolchains=;python3;' \
             -t duckdb/${{ inputs.duckdb_arch }} \
             ./extension-ci-tools/docker/${{ inputs.duckdb_arch }}
@@ -167,25 +167,3 @@ runs:
         shell: bash
         run: |
           docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make test_release
-
-      - name: Deploy
-        if: ${{ inputs.deploy_as != '' }}
-        shell: bash
-        env:
-          AWS_ACCESS_KEY_ID: ${{ inputs.s3_id }}
-          AWS_SECRET_ACCESS_KEY: ${{ inputs.s3_key }}
-          DUCKDB_EXTENSION_SIGNING_PK: ${{ inputs.signing_pk }}
-          AWS_DEFAULT_REGION: us-east-1
-          DUCKDB_DEPLOY_SCRIPT_MODE: for_real
-        run: |
-          cd  ${{ inputs.build_dir}}
-          if [[ "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
-            if [[ ! -z "${{ inputs.deploy_version }}" ]] ; then
-              duckdb/scripts/extension-upload-all.sh ${{ inputs.deploy_as }} ${{ inputs.deploy_version }}
-            elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-              duckdb/scripts/extension-upload-all.sh ${{ inputs.deploy_as }} ${{ github.ref_name }}
-            elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-              duckdb/scripts/extension-upload-all.sh ${{ inputs.deploy_as }} "${GITHUB_SHA:0:10}"
-            fi
-          fi
-

.github/actions/manylinux_2014_setup/action.yml

@@ -96,7 +96,7 @@ runs:
       if: ${{ inputs.vcpkg == 1 }}
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+        vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
     - name: Install OpenSSL
       if: ${{ inputs.openssl == 1 }}

.github/actions/ubuntu_18_setup/action.yml

@@ -88,7 +88,7 @@ runs:
       if: ${{ inputs.vcpkg == 1 }}
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+        vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
 
     - name: Setup Ccache
       if: ${{ inputs.ccache == 1 }}

.github/config/extensions.csv

@@ -12,7 +12,7 @@ substrait,https://github.com/duckdb/substrait,1116fb580edd3e26e675436dbdbdf4a0aa
 arrow,https://github.com/duckdb/arrow,9e10240da11f61ea7fbfe3fc9988ffe672ccd40f,no-windows
 aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb97,
 azure,https://github.com/duckdb/duckdb_azure,09623777a366572bfb8fa53e47acdf72133a360e,
-spatial,https://github.com/duckdb/duckdb_spatial,7ea79b614755d2bdee4be468691e4e17b39b8dbc,
+spatial,https://github.com/duckdb/duckdb_spatial,a60aa3733741a99c49baaf33390c0f7c8a9598a3,
 iceberg,https://github.com/duckdb/duckdb_iceberg,d89423c2ff90a0b98a093a133c8dfe2a55b9e092,
 vss,https://github.com/duckdb/duckdb_vss,96374099476b3427c9ab43c1821e610b0465c864,
 fts,https://github.com/duckdb/duckdb_fts,0477abaf2484aa7b9aabf8ace9dc0bde80a15554,

.github/config/in_tree_extensions.cmake

@@ -7,7 +7,6 @@
 
 duckdb_extension_load(autocomplete)
 duckdb_extension_load(core_functions)
-duckdb_extension_load(httpfs)
 duckdb_extension_load(icu)
 duckdb_extension_load(json)
 duckdb_extension_load(parquet)

.github/config/out_of_tree_extensions.cmake

@@ -15,8 +15,16 @@
 #  VCPKG_TOOLCHAIN_PATH=~/vcpkg/scripts/buildsystems/vcpkg.cmake
 #  VCPKG_TARGET_TRIPLET=arm64-osx
 
+################# HTTPFS
+duckdb_extension_load(httpfs
+    LOAD_TESTS
+    GIT_URL https://github.com/duckdb/duckdb_httpfs
+    GIT_TAG 92867fce5a6e90b9a6803fab2b196c00cc11e46c
+    INCLUDE_DIR extension/httpfs/include
+    )
+
 ################# ARROW
-if (NOT MINGW)
+if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(arrow
             LOAD_TESTS DONT_LINK
             GIT_URL https://github.com/duckdb/arrow
@@ -26,11 +34,12 @@ if (NOT MINGW)
 endif()
 
 ################## AWS
-if (NOT MINGW)
+if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(aws
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_aws
             GIT_TAG f743d4b3c2faecda15498d0219a1727ad6d62b5b
+            APPLY_PATCHES
             )
 endif()
 
@@ -39,7 +48,7 @@ endif()
 ### build on a side
 if (NO)
 ################# AZURE
-if (NOT MINGW)
+if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(azure
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_azure
@@ -52,7 +61,7 @@ endif()
 ################# DELTA
 # MinGW build is not available, and our current manylinux ci does not have enough storage space to run the rust build
 # for Delta
-if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux")
+if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux" AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(delta
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_delta
@@ -65,26 +74,26 @@ endif()
 duckdb_extension_load(excel
     LOAD_TESTS
     GIT_URL https://github.com/duckdb/duckdb_excel
-    GIT_TAG 0e99dc789038c7af658e30d579b818473a6d6ea8
-    INCLUDE_DIR extension/excel/include
+    GIT_TAG e243577956f36a898d5485189e5288839c2c4b73
+    INCLUDE_DIR src/excel/include
     )
 
 ################# ICEBERG
 # Windows tests for iceberg currently not working
-if (NOT WIN32)
-    set(LOAD_ICEBERG_TESTS "LOAD_TESTS")
-else ()
-    set(LOAD_ICEBERG_TESTS "")
-endif()
-
-if (NOT MINGW)
-    duckdb_extension_load(iceberg
-            ${LOAD_ICEBERG_TESTS}
-            GIT_URL https://github.com/duckdb/duckdb_iceberg
-            GIT_TAG d62d91d8a089371c4d1862a88f2e62a97bc2af3a
-            APPLY_PATCHES
-            )
-endif()
+#if (NOT WIN32)
+#    set(LOAD_ICEBERG_TESTS "LOAD_TESTS")
+#else ()
+#    set(LOAD_ICEBERG_TESTS "")
+#endif()
+#
+#if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
+#    duckdb_extension_load(iceberg
+#            ${LOAD_ICEBERG_TESTS}
+#            GIT_URL https://github.com/duckdb/duckdb_iceberg
+#            GIT_TAG d62d91d8a089371c4d1862a88f2e62a97bc2af3a
+#            APPLY_PATCHES
+#            )
+#endif()
 
 ################# INET
 duckdb_extension_load(inet
@@ -93,12 +102,13 @@ duckdb_extension_load(inet
     GIT_TAG 51d7ad789f34eecb36a2071bac5aef0e12747d70
     INCLUDE_DIR src/include
     TEST_DIR test/sql
+    APPLY_PATCHES
     )
 
 ################# POSTGRES_SCANNER
 # Note: tests for postgres_scanner are currently not run. All of them need a postgres server running. One test
 #       uses a remote rds server but that's not something we want to run here.
-if (NOT MINGW)
+if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(postgres_scanner
             DONT_LINK
             GIT_URL https://github.com/duckdb/postgres_scanner
@@ -107,15 +117,17 @@ if (NOT MINGW)
             )
 endif()
 
+# mingw CI with all extensions at once is somehow not happy
+if (NOT MINGW)
 ################# SPATIAL
 duckdb_extension_load(spatial
     DONT_LINK LOAD_TESTS
     GIT_URL https://github.com/duckdb/duckdb_spatial.git
-    GIT_TAG 7ea79b614755d2bdee4be468691e4e17b39b8dbc
+    GIT_TAG a60aa3733741a99c49baaf33390c0f7c8a9598a3
     INCLUDE_DIR spatial/include
     TEST_DIR test/sql
-    APPLY_PATCHES
     )
+endif()
 
 ################# SQLITE_SCANNER
 # Static linking on windows does not properly work due to symbol collision
@@ -132,11 +144,13 @@ duckdb_extension_load(sqlite_scanner
         APPLY_PATCHES
         )
 
+if (NOT $ENV{WASM_EXTENSIONS})
 duckdb_extension_load(sqlsmith
         DONT_LINK LOAD_TESTS
         GIT_URL https://github.com/duckdb/duckdb_sqlsmith
         GIT_TAG d6d62c1cba6b1369ba79db4bff3c67f24aaa95c2
         )
+endif()
 
 ################# VSS
 duckdb_extension_load(vss
@@ -149,7 +163,7 @@ duckdb_extension_load(vss
     )
 
 ################# MYSQL
-if (NOT MINGW)
+if (NOT MINGW AND NOT $ENV{WASM_EXTENSIONS})
     duckdb_extension_load(mysql_scanner
             DONT_LINK
             LOAD_TESTS

.github/patches/extensions/aws/require_autoloading.patch

@@ -0,0 +1,20 @@
+diff --git a/test/sql/aws_errors.test b/test/sql/aws_errors.test
+index 50fb7d7..292724e 100644
+--- a/test/sql/aws_errors.test
++++ b/test/sql/aws_errors.test
+@@ -2,7 +2,7 @@
+ # description: test aws extension
+ # group: [aws]
+ 
+-require no_extension_autoloading
++require no_extension_autoloading "EXPECTED: Test relies on autoloading not to be there"
+ 
+ # Before we load the extension, this will fail
+ statement error
+@@ -22,4 +22,4 @@ httpfs extension is required for load_aws_credentials
+ require httpfs
+ 
+ statement ok
+-CALL load_aws_credentials();
+\ No newline at end of file
++CALL load_aws_credentials();

.github/patches/extensions/inet/require_autoloading.patch

@@ -0,0 +1,26 @@
+diff --git a/test/sql/test_inet_storage.test b/test/sql/test_inet_storage.test
+index 6679439..7de7620 100644
+--- a/test/sql/test_inet_storage.test
++++ b/test/sql/test_inet_storage.test
+@@ -5,7 +5,7 @@
+ require inet
+ 
+ # FIXME: Make inet properly autoloadable
+-require no_extension_autoloading
++require no_extension_autoloading "FIXME: to be reviewed whether this can be lifted"
+ 
+ # load the DB from disk
+ load __TEST_DIR__/store_inet.db
+diff --git a/test/sql/test_ipv6_inet_storage.test b/test/sql/test_ipv6_inet_storage.test
+index 5be1ff2..7dc1930 100644
+--- a/test/sql/test_ipv6_inet_storage.test
++++ b/test/sql/test_ipv6_inet_storage.test
+@@ -25,7 +25,7 @@ CREATE VIEW iview AS SELECT INET '::1'
+ restart
+ 
+ #FIXME: INET needs to be explicitly autoloaded on restart
+-require no_extension_autoloading
++require no_extension_autoloading "FIXME: INET needs to be explicitly autoloaded on restart"
+ 
+ query IIIIII
+ DESCRIBE tbl