Merge pull request #49 from quentingodeau/feature/hierarchical-namespace

Feature/hierarchical namespace

.github/workflows/LocalTesting.yml

@@ -17,6 +17,7 @@ jobs:
       GEN: Ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
+      AZURE_STORAGE_ACCOUNT: devstoreaccount1
       HTTP_PROXY_RUNNING: '1'
 
     steps:
@@ -87,6 +88,7 @@ jobs:
       GEN: Ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
+      AZURE_STORAGE_ACCOUNT: devstoreaccount1
       HTTP_PROXY_RUNNING: '1'
 
     steps:
@@ -161,6 +163,7 @@ jobs:
       VCPKG_ROOT: ${{ github.workspace }}\vcpkg
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake
       AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
+      AZURE_STORAGE_ACCOUNT: devstoreaccount1
 
     steps:
       - uses: actions/checkout@v3

CMakeLists.txt

@@ -11,33 +11,45 @@ set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 set(EXTENSION_SOURCES
-    src/azure_extension.cpp src/azure_secret.cpp
-    src/azure_storage_account_client.cpp src/azure_filesystem.cpp
-    src/http_state_policy.cpp)
+    src/azure_extension.cpp
+    src/azure_secret.cpp
+    src/azure_filesystem.cpp
+    src/azure_storage_account_client.cpp
+    src/azure_blob_filesystem.cpp
+    src/azure_dfs_filesystem.cpp
+    src/http_state_policy.cpp
+    src/azure_parsed_url.cpp)
 add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES})
 
 set(PARAMETERS "-warnings")
 build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES})
 
 find_package(azure-identity-cpp CONFIG)
 find_package(azure-storage-blobs-cpp CONFIG)
+find_package(azure-storage-files-datalake-cpp CONFIG)
 
-if(NOT ${azure-identity-cpp_FOUND} OR NOT ${azure-storage-blobs-cpp_FOUND})
+if(NOT ${azure-identity-cpp_FOUND}
+   OR NOT ${azure-storage-blobs-cpp_FOUND}
+   OR NOT ${azure-storage-files-datalake-cpp_FOUND})
   message(FATAL_ERROR "Azure SDK not found, did you set up vcpkg correctly?")
 endif()
 
 # Static lib
-target_link_libraries(${EXTENSION_NAME} Azure::azure-identity
-                      Azure::azure-storage-blobs)
-target_include_directories(${EXTENSION_NAME} PRIVATE Azure::azure-identity
-                                                     Azure::azure-storage-blobs)
+target_link_libraries(
+  ${EXTENSION_NAME} Azure::azure-identity Azure::azure-storage-blobs
+  Azure::azure-storage-files-datalake)
+target_include_directories(
+  ${EXTENSION_NAME} PRIVATE Azure::azure-identity Azure::azure-storage-blobs
+                            Azure::azure-storage-files-datalake)
 
 # Loadable binary
-target_link_libraries(${TARGET_NAME}_loadable_extension Azure::azure-identity
-                      Azure::azure-storage-blobs)
+target_link_libraries(
+  ${TARGET_NAME}_loadable_extension Azure::azure-identity
+  Azure::azure-storage-blobs Azure::azure-storage-files-datalake)
 target_include_directories(
-  ${TARGET_NAME}_loadable_extension PRIVATE Azure::azure-identity
-                                            Azure::azure-storage-blobs)
+  ${TARGET_NAME}_loadable_extension
+  PRIVATE Azure::azure-identity Azure::azure-storage-blobs
+          Azure::azure-storage-files-datalake)
 
 install(
   TARGETS ${EXTENSION_NAME}

data/README.md

@@ -1,3 +1,20 @@
 # Test data
+
 This directory contains test data that is uploaded to Azure tests servers in CI. What this means is that when adding
-files in this directory, the `test/sql/test_data_integrity.test` should be updated, otherwise CI will fail.
+files in this directory, the `test/sql/test_data_integrity.test` should be updated, otherwise CI will fail (see [gen_check_file script](../scripts/gen_check_file.sh)).
+
+## Partitioned
+
+Partitionned data has been add to test the integration of DFS storage account.
+
+Data has been generated from the `l.csv` source file with the following code:
+
+```sql
+COPY (
+    SELECT *, date_part('year', l_receiptdate) as l_receipmonth
+    FROM './data/l.csv'
+    WHERE l_receipmonth >= 1997
+    AND   l_shipmode IN ('AIR', 'SHIP', 'TRUCK')
+)
+TO './data/partitioned' (FORMAT CSV, PARTITION_BY (l_receipmonth, l_shipmode), OVERWRITE_OR_IGNORE 1);
+```