feat: Add NewListDirectoryPathsPager for azuredatalake (#23905)

* feat: Add NewListDirectoryPathsPager for azuredatalake

* remove TestFilesystemListDirectoryPathsContinuation

sdk/storage/azdatalake/CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Breaking Changes
 
 ### Bugs Fixed
+* Added NewListDirectoryPathPager. Fixes [#23852](https://github.com/Azure/azure-sdk-for-go/issues/23852), [#21083](https://github.com/Azure/azure-sdk-for-go/issues/21083), [#18921](https://github.com/Azure/azure-sdk-for-go/issues/18921)
 
 ### Other Changes
 

sdk/storage/azdatalake/filesystem/client.go

@@ -4,12 +4,16 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-// FOR FS CLIENT WE STORE THE GENERATED DATALAKE LAYER WITH BLOB ENDPOINT IN ORDER TO USE DELETED PATH LISTING
+// FOR FS CLIENT WE STORE THE GENERATED DATALAKE LAYER WITH BLOB ENDPOINT IN ORDER TO USE DELETED/DIRECTORY PATH LISTING
 
 package filesystem
 
 import (
 	"context"
+	"net/http"
+	"strings"
+	"time"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
@@ -23,9 +27,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
-	"net/http"
-	"strings"
-	"time"
 )
 
 // ClientOptions contains the optional parameters when creating a Client.
@@ -307,6 +308,41 @@ func (fs *Client) NewListPathsPager(recursive bool, options *ListPathsOptions) *
 	})
 }
 
+// NewListDirectoryPathsPager operation returns a pager of the directory paths under the specified filesystem.
+func (fs *Client) NewListDirectoryPathsPager(options *ListDirectoryPathsOptions) *runtime.Pager[ListDirectoryPathsSegmentResponse] {
+	listOptions := options.format()
+	return runtime.NewPager(runtime.PagingHandler[ListDirectoryPathsSegmentResponse]{
+		More: func(page ListDeletedPathsSegmentResponse) bool {
+			return page.NextMarker != nil && len(*page.NextMarker) > 0
+		},
+		Fetcher: func(ctx context.Context, page *ListDirectoryPathsSegmentResponse) (ListDirectoryPathsSegmentResponse, error) {
+			var req *policy.Request
+			var err error
+			if page == nil {
+				req, err = fs.generatedFSClientWithBlob().ListBlobHierarchySegmentCreateRequest(ctx, &listOptions)
+				err = exported.ConvertToDFSError(err)
+			} else {
+				listOptions.Marker = page.NextMarker
+				req, err = fs.generatedFSClientWithBlob().ListBlobHierarchySegmentCreateRequest(ctx, &listOptions)
+				err = exported.ConvertToDFSError(err)
+			}
+			if err != nil {
+				return ListDirectoryPathsSegmentResponse{}, err
+			}
+			resp, err := fs.generatedFSClientWithBlob().InternalClient().Pipeline().Do(req)
+			err = exported.ConvertToDFSError(err)
+			if err != nil {
+				return ListDirectoryPathsSegmentResponse{}, err
+			}
+			if !runtime.HasStatusCode(resp, http.StatusOK) {
+				return ListDirectoryPathsSegmentResponse{}, runtime.NewResponseError(resp)
+			}
+			newResp, err := fs.generatedFSClientWithBlob().ListBlobHierarchySegmentHandleResponse(resp)
+			return newResp, exported.ConvertToDFSError(err)
+		},
+	})
+}
+
 // NewListDeletedPathsPager operation returns a pager of the shares under the specified account.
 // For more information, see https://learn.microsoft.com/en-us/rest/api/storageservices/list-shares
 func (fs *Client) NewListDeletedPathsPager(options *ListDeletedPathsOptions) *runtime.Pager[ListDeletedPathsSegmentResponse] {

sdk/storage/azdatalake/filesystem/client_test.go

@@ -8,12 +8,13 @@ package filesystem_test
 
 import (
 	"context"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/internal/recording"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/datalakeerror"
@@ -1730,6 +1731,107 @@ func (s *RecordedTestSuite) TestFilesystemListPathsWithEncryptionContext() {
 	}
 }
 
+func (s *UnrecordedTestSuite) TestFilesystemListDirectoryPaths() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirClient := fsClient.NewDirectoryClient(testName + "dir1")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	dirClient = fsClient.NewDirectoryClient(testName + "dir2")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	pager := fsClient.NewListDirectoryPathsPager(nil)
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		_require.NoError(err)
+		_require.Equal(3, len(resp.ListPathsHierarchySegmentResponse.Segment.PathItems))
+		if err != nil {
+			break
+		}
+	}
+}
+
+func (s *UnrecordedTestSuite) TestFilesystemListDirectoryPathsMaxResults() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirClient := fsClient.NewDirectoryClient(testName + "dir1")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	dirClient = fsClient.NewDirectoryClient(testName + "dir2")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	pages := 3
+	count := 0
+	opts := filesystem.ListDirectoryPathsOptions{
+		MaxResults: to.Ptr(int32(1)),
+	}
+
+	pager := fsClient.NewListDirectoryPathsPager(&opts)
+	for pager.More() {
+		_, err := pager.NextPage(context.Background())
+		_require.NoError(err)
+		count += 1
+		if err != nil {
+			break
+		}
+	}
+	_require.Equal(pages, count)
+}
+
+func (s *UnrecordedTestSuite) TestFilesystemListDirectoryPathsWithPrefix() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirClient := fsClient.NewDirectoryClient(testName + "dir1")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	dirClient = fsClient.NewDirectoryClient(testName + "dir2")
+	_, err = dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	opts := filesystem.ListDirectoryPathsOptions{
+		Prefix: to.Ptr("Test"),
+	}
+
+	pager := fsClient.NewListDirectoryPathsPager(&opts)
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		_require.NoError(err)
+		_require.Equal(3, len(resp.ListPathsHierarchySegmentResponse.Segment.PathItems))
+		if err != nil {
+			break
+		}
+	}
+}
+
 func (s *RecordedTestSuite) TestFilesystemListDeletedPaths() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azdatalake/filesystem/examples_test.go

@@ -10,14 +10,15 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
@@ -182,6 +183,33 @@ func Example_fs_ClientListPaths() {
 	}
 }
 
+func Example_fs_ClientListDirectoryPaths() {
+	accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
+	if !ok {
+		panic("AZURE_STORAGE_ACCOUNT_NAME could not be found")
+	}
+	fsName := "testfs"
+	fsURL := fmt.Sprintf("https://%s.dfs.core.windows.net/%s", accountName, fsName)
+
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	handleError(err)
+
+	fsClient, err := filesystem.NewClient(fsURL, cred, nil)
+	handleError(err)
+
+	pager := fsClient.NewListDirectoryPathsPager(nil)
+
+	for pager.More() {
+		resp, err := pager.NextPage(context.TODO())
+		if err != nil {
+			log.Fatal(err)
+		}
+		for _, path := range resp.Segment.PathItems {
+			fmt.Println(*path.Name)
+		}
+	}
+}
+
 func Example_fs_ClientListDeletedPaths() {
 	accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
 	if !ok {

sdk/storage/azdatalake/filesystem/models.go

@@ -7,12 +7,13 @@
 package filesystem
 
 import (
+	"time"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/file"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
-	"time"
 )
 
 // SetAccessPolicyOptions provides set of configurations for FileSystem.SetAccessPolicy operation.
@@ -158,6 +159,31 @@ func (o *ListPathsOptions) format() generated.FileSystemClientListPathsOptions {
 	}
 }
 
+// ListDirectoryPathsOptions contains the optional parameters from the FileSystem.ListDirectoryPathsOptions.
+type ListDirectoryPathsOptions struct {
+	// Marker contains last continuation token returned from the service for listing.
+	Marker *string
+	// MaxResults sets the maximum number of paths that will be returned per page.
+	MaxResults *int32
+	// Prefix filters the results to return only paths whose names begin with the specified prefix path.
+	Prefix *string
+}
+
+func (o *ListDirectoryPathsOptions) format() generated.FileSystemClientListBlobHierarchySegmentOptions {
+	showOnly := generated.ListBlobsShowOnlyDirectories
+	if o == nil {
+		return generated.FileSystemClientListBlobHierarchySegmentOptions{
+			Showonly: &showOnly,
+		}
+	}
+	return generated.FileSystemClientListBlobHierarchySegmentOptions{
+		Marker:     o.Marker,
+		MaxResults: o.MaxResults,
+		Prefix:     o.Prefix,
+		Showonly:   &showOnly,
+	}
+}
+
 // ListDeletedPathsOptions contains the optional parameters for the FileSystem.ListDeletedPaths operation.
 type ListDeletedPathsOptions struct {
 	// Marker contains last continuation token returned from the service for listing.
@@ -169,7 +195,7 @@ type ListDeletedPathsOptions struct {
 }
 
 func (o *ListDeletedPathsOptions) format() generated.FileSystemClientListBlobHierarchySegmentOptions {
-	showOnly := "deleted"
+	showOnly := generated.ListBlobsShowOnlyDeleted
 	if o == nil {
 		return generated.FileSystemClientListBlobHierarchySegmentOptions{Showonly: &showOnly}
 	}

sdk/storage/azdatalake/filesystem/responses.go

@@ -7,10 +7,11 @@
 package filesystem
 
 import (
+	"time"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
-	"time"
 )
 
 // GetAccessPolicyResponse contains the response from method FileSystemClient.GetAccessPolicy.
@@ -145,6 +146,9 @@ type UndeletePathResponse = generated.PathClientUndeleteResponse
 // ListDeletedPathsSegmentResponse contains the response from method FileSystemClient.ListPathsSegment.
 type ListDeletedPathsSegmentResponse = generated.FileSystemClientListPathHierarchySegmentResponse
 
+// ListDirectoryPathsSegmentResponse contains the response from method FileSystemClient.ListDirectoryPathsSegmentResponse.
+type ListDirectoryPathsSegmentResponse = generated.FileSystemClientListPathHierarchySegmentResponse
+
 // ListPathsHierarchySegmentResponse contains the response from method FileSystemClient.ListPathsHierarchySegment.
 type ListPathsHierarchySegmentResponse = generated.ListPathsHierarchySegmentResponse
 

sdk/storage/azdatalake/internal/generated/autorest.md

@@ -22,6 +22,18 @@ export-clients: true
 use: "@autorest/[email protected]"
 ```
 
+### Add ListBlobsShowOnly value 'directories'
+```yaml
+directive:
+- from: swagger-document
+  where: $.parameters.ListBlobsShowOnly
+  transform: >
+    if (!$.enum.includes("directories")) {
+        $.enum.push("directories");
+    }
+
+```
+
 ### Remove FileSystem and PathName from parameter list since they are not needed
 ``` yaml
 directive: