[Test run] Dev/jetyson/azure

build/config.props

@@ -3,7 +3,7 @@
   <!-- Dependency versions -->
   <PropertyGroup>
     <NuGetPackageVersion>6.9.1</NuGetPackageVersion>
-    <MicrosoftAzureStorageBlobVersion>11.2.3</MicrosoftAzureStorageBlobVersion>
+    <AzureStorageBlobsVersion>12.19.1</AzureStorageBlobsVersion>
     <JsonVersion>13.0.3</JsonVersion>
     <CommandLineUtilsVersion>2.0.0</CommandLineUtilsVersion>
     <NuGetTestHelpersVersion>2.1.36</NuGetTestHelpersVersion>

doc/ci-server.md

@@ -10,7 +10,7 @@ Github Action is free and common CI/CD tools to developer.
 
 Now we take example to publish some nuget package in Github Action.
 
-You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs and etc.
+You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs, etc.
 
 Now, we are going to publish nuget package to Azure by Github Action.
 
@@ -60,10 +60,10 @@ It means:
 
 1. It will run CI action in ubuntu-latest OS
 2. Step named `pack` will `cd` to the `src` directory and try to pack nuget packages versioned `1.0.0`
-3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` diretocy to azure
+3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` directory to azure
 4. \${{secrets.SLEET_CONNECTIONSTRING}} is a secret, you can add it in you repository settings tab.
 
-Put it togather as below:
+Put it together as below:
 
 ```yml
 name: Publish dev nuget package to azure
@@ -99,7 +99,7 @@ jobs:
 
 ### Some tips
 
-#### Please pack nuget package to out it to specfic directory
+#### Please pack nuget package to out it to specific directory
 
 Maybe you publish nuget package via `dotnet` or `nuget` command as script below:
 
@@ -109,4 +109,4 @@ dotnet nuget push **/*.nupkg
 
 Unfortunately, It is not support to publish package via `sleet push **/*.nupkg`.
 
-So, please pack your nuget package to some directory like `pkg` as exmaple above.
+So, please pack your nuget package to some directory like `pkg` as example above.

src/SleetLib/FileSystem/AzureBlobLease.cs

@@ -1,24 +1,21 @@
-using System;
+using Azure.Storage.Blobs.Specialized;
+using Azure.Storage.Blobs;
 using System.Diagnostics;
-using System.Threading.Tasks;
-using Microsoft.Azure.Storage;
-using Microsoft.Azure.Storage.Blob;
 
 namespace Sleet
 {
     public class AzureBlobLease : IDisposable
     {
         private static readonly TimeSpan _leaseTime = new TimeSpan(0, 1, 0);
-        private readonly CloudBlockBlob _blob;
-        private readonly string _leaseId;
+        private readonly BlobClient _blob;
 
-        public AzureBlobLease(CloudBlockBlob blob)
+        public AzureBlobLease(BlobClient blob)
         {
             _blob = blob;
-            _leaseId = Guid.NewGuid().ToString();
+            LeaseId = Guid.NewGuid().ToString();
         }
 
-        public string LeaseId => _leaseId;
+        public string LeaseId { get; }
 
         /// <summary>
         /// Makes a single attempt to get a lease.
@@ -30,21 +27,21 @@ public async Task<bool> GetLease()
 
             try
             {
-                actualLease = await _blob.AcquireLeaseAsync(_leaseTime, _leaseId);
+                actualLease = (await _blob.GetBlobLeaseClient(LeaseId).AcquireAsync(_leaseTime)).Value.LeaseId;
             }
             catch (Exception ex)
             {
                 Debug.Fail($"GetLease failed: {ex}");
             }
 
-            return StringComparer.Ordinal.Equals(_leaseId, actualLease);
+            return StringComparer.Ordinal.Equals(LeaseId, actualLease);
         }
 
         public async Task Renew()
         {
             try
             {
-                await _blob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId));
+                await _blob.GetBlobLeaseClient(LeaseId).RenewAsync();
             }
             catch (Exception ex)
             {
@@ -63,7 +60,8 @@ public void Release()
         {
             try
             {
-                _blob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId)).Wait(TimeSpan.FromSeconds(60));
+
+                _blob.GetBlobLeaseClient(LeaseId).ReleaseAsync().Wait(TimeSpan.FromSeconds(60));
             }
             catch (Exception ex)
             {
@@ -72,4 +70,4 @@ public void Release()
             }
         }
     }
-}
+}

src/SleetLib/FileSystem/AzureFile.cs

@@ -1,38 +1,36 @@
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Azure.Storage.Blob;
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Blobs;
 using NuGet.Common;
+using System.IO.Compression;
 
 namespace Sleet
 {
     public class AzureFile : FileBase
     {
-        private readonly CloudBlockBlob _blob;
+        private readonly BlobClient _blob;
 
-        internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, CloudBlockBlob blob)
+        internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, BlobClient blob)
             : base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
         {
             _blob = blob;
         }
 
         protected override async Task CopyFromSource(ILogger log, CancellationToken token)
         {
-            if (await _blob.ExistsAsync())
+            if (await _blob.ExistsAsync(token))
             {
                 log.LogVerbose($"GET {_blob.Uri.AbsoluteUri}");
 
                 DeleteInternal();
 
                 using (var cache = File.OpenWrite(LocalCacheFile.FullName))
                 {
-                    await _blob.DownloadToStreamAsync(cache);
+                    await _blob.DownloadToAsync(cache, token);
                 }
 
                 // If the blob is compressed it needs to be decompressed locally before it can be used
-                if (_blob.Properties.ContentEncoding?.Equals("gzip", StringComparison.OrdinalIgnoreCase) == true)
+                var blobProperties = await _blob.GetPropertiesAsync(cancellationToken: token);
+                if (blobProperties.Value.ContentEncoding != null && blobProperties.Value.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase))
                 {
                     log.LogVerbose($"Decompressing {_blob.Uri.AbsoluteUri}");
 
@@ -60,71 +58,68 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
                 using (var cache = LocalCacheFile.OpenRead())
                 {
                     Stream writeStream = cache;
+                    var blobHeaders = new BlobHttpHeaders
+                    {
+                        CacheControl = "no-store"
+                    };
 
                     if (_blob.Uri.AbsoluteUri.EndsWith(".nupkg", StringComparison.Ordinal))
                     {
-                        _blob.Properties.ContentType = "application/zip";
+                        blobHeaders.ContentType = "application/zip";
                     }
                     else if (_blob.Uri.AbsoluteUri.EndsWith(".xml", StringComparison.Ordinal)
                         || _blob.Uri.AbsoluteUri.EndsWith(".nuspec", StringComparison.Ordinal))
                     {
-                        _blob.Properties.ContentType = "application/xml";
+                        blobHeaders.ContentType = "application/xml";
                     }
                     else if (_blob.Uri.AbsoluteUri.EndsWith(".svg", StringComparison.Ordinal))
                     {
-                        _blob.Properties.ContentType = "image/svg+xml";
+                        blobHeaders.ContentType = "image/svg+xml";
                     }
                     else if (_blob.Uri.AbsoluteUri.EndsWith(".json", StringComparison.Ordinal)
                             || await JsonUtility.IsJsonAsync(LocalCacheFile.FullName))
                     {
-                        _blob.Properties.ContentType = "application/json";
+                        blobHeaders.ContentType = "application/json";
 
                         if (!SkipCompress())
                         {
-                            _blob.Properties.ContentEncoding = "gzip";
+                            blobHeaders.ContentEncoding = "gzip";
                             writeStream = await JsonUtility.GZipAndMinifyAsync(cache);
                         }
                     }
                     else if (_blob.Uri.AbsoluteUri.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)
                         || _blob.Uri.AbsoluteUri.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase))
                     {
-                        _blob.Properties.ContentType = "application/octet-stream";
+                        blobHeaders.ContentType = "application/octet-stream";
                     }
                     else if (_blob.Uri.AbsoluteUri.EndsWith("/icon"))
                     {
-                        _blob.Properties.ContentType = "image/png";
+                        blobHeaders.ContentType = "image/png";
                     }
                     else
                     {
                         log.LogWarning($"Unknown file type: {_blob.Uri.AbsoluteUri}");
                     }
 
-                    await _blob.UploadFromStreamAsync(writeStream);
+                    await _blob.UploadAsync(writeStream, blobHeaders, cancellationToken: token);
 
                     writeStream.Dispose();
                 }
-
-                _blob.Properties.CacheControl = "no-store";
-
-                // TODO: re-enable this once it works again.
-                _blob.Properties.ContentMD5 = null;
-
-                await _blob.SetPropertiesAsync();
             }
-            else if (await _blob.ExistsAsync())
+            else if (await _blob.ExistsAsync(token))
             {
                 log.LogVerbose($"Removing {_blob.Uri.AbsoluteUri}");
-                await _blob.DeleteAsync();
+                await _blob.DeleteAsync(cancellationToken: token);
             }
             else
             {
                 log.LogVerbose($"Skipping {_blob.Uri.AbsoluteUri}");
             }
         }
 
-        protected override Task<bool> RemoteExists(ILogger log, CancellationToken token)
+        protected override async Task<bool> RemoteExists(ILogger log, CancellationToken token)
         {
-            return _blob.ExistsAsync();
+            return await _blob.ExistsAsync(token);
         }
     }
-}
+}

src/SleetLib/FileSystem/AzureFileSystem.cs

@@ -1,33 +1,24 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Azure.Storage;
-using Microsoft.Azure.Storage.Blob;
+using Azure.Storage.Blobs;
 using NuGet.Common;
+using Azure.Storage.Blobs.Models;
 
 namespace Sleet
 {
     public class AzureFileSystem : FileSystemBase
     {
         public static readonly string AzureEmptyConnectionString = "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;BlobEndpoint=";
 
-        private readonly CloudStorageAccount _azureAccount;
-        private readonly CloudBlobClient _client;
-        private readonly CloudBlobContainer _container;
+        private readonly BlobContainerClient _container;
 
-        public AzureFileSystem(LocalCache cache, Uri root, CloudStorageAccount azureAccount, string container)
-            : this(cache, root, root, azureAccount, container)
+        public AzureFileSystem(LocalCache cache, Uri root, BlobServiceClient blobServiceClient, string container)
+            : this(cache, root, root, blobServiceClient, container)
         {
         }
 
-        public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, CloudStorageAccount azureAccount, string container, string feedSubPath = null)
+        public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, BlobServiceClient blobServiceClient, string container, string feedSubPath = null)
             : base(cache, root, baseUri, feedSubPath)
         {
-            _azureAccount = azureAccount;
-            _client = _azureAccount.CreateCloudBlobClient();
-            _container = _client.GetContainerReference(container);
+            _container = blobServiceClient.GetBlobContainerClient(container);
 
             var containerUri = UriUtility.EnsureTrailingSlash(_container.Uri);
             var expectedPath = UriUtility.EnsureTrailingSlash(root);
@@ -63,14 +54,14 @@ public override ISleetFile Get(Uri path)
                     pair.Root,
                     pair.BaseURI,
                     LocalCache.GetNewTempPath(),
-                    _container.GetBlockBlobReference(GetRelativePath(path))));
+                    _container.GetBlobClient(GetRelativePath(path))));
         }
 
         public override async Task<bool> Validate(ILogger log, CancellationToken token)
         {
             log.LogInformation($"Verifying {_container.Uri.AbsoluteUri} exists.");
 
-            if (await _container.ExistsAsync())
+            if (await _container.ExistsAsync(token))
             {
                 log.LogInformation($"Found {_container.Uri.AbsoluteUri}");
             }
@@ -86,34 +77,32 @@ public override async Task<bool> Validate(ILogger log, CancellationToken token)
         public override ISleetFileSystemLock CreateLock(ILogger log)
         {
             // Create blobs
-            var blob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
-            var messageBlob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
+            var blob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
+            var messageBlob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
             return new AzureFileSystemLock(blob, messageBlob, log);
         }
 
         public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, CancellationToken token)
         {
-            string prefix = null;
-            var useFlatBlobListing = true;
-            var blobListingDetails = BlobListingDetails.All;
-            int? maxResults = null;
+            var results = _container.GetBlobsAsync();
+            var pages = results.AsPages();
+            var blobs = new List<ISleetFile>();
 
-            // Return all files except feedlock
-            var blobs = new List<IListBlobItem>();
-
-            BlobResultSegment result = null;
-            do
+            await foreach (var page in pages)
             {
-                result = await _container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, result?.ContinuationToken, options: null, operationContext: null);
-                blobs.AddRange(result.Results);
+                // process page
+                blobs.AddRange(
+                    page.Values
+                        .Where(item => !item.Name.EndsWith(AzureFileSystemLock.LockFile, StringComparison.Ordinal))
+                        .Where(item =>
+                            string.IsNullOrEmpty(FeedSubPath) ||
+                            item.Name.StartsWith(FeedSubPath, StringComparison.Ordinal))
+                        .Select(item =>
+                            Get(new BlobUriBuilder(_container.Uri) { BlobName = item.Name }.ToUri())
+                            ));
             }
-            while (result.ContinuationToken != null);
 
-            // Skip the feed lock, and limit this to the current sub feed.
-            return blobs.Where(e => !e.Uri.AbsoluteUri.EndsWith($"/{AzureFileSystemLock.LockFile}"))
-                 .Where(e => string.IsNullOrEmpty(FeedSubPath) || e.Uri.AbsoluteUri.StartsWith(UriUtility.EnsureTrailingSlash(BaseURI).AbsoluteUri, StringComparison.Ordinal))
-                 .Select(e => Get(e.Uri))
-                 .ToList();
+            return blobs;
         }
 
         public override string GetRelativePath(Uri uri)
@@ -128,19 +117,19 @@ public override string GetRelativePath(Uri uri)
             return relativePath;
         }
 
-        public override Task<bool> HasBucket(ILogger log, CancellationToken token)
+        public override async Task<bool> HasBucket(ILogger log, CancellationToken token)
         {
-            return _container.ExistsAsync(token);
+            return await _container.ExistsAsync(token);
         }
 
         public override async Task CreateBucket(ILogger log, CancellationToken token)
         {
-            await _container.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Blob, null, null, token);
+            await _container.CreateIfNotExistsAsync(PublicAccessType.BlobContainer, null, null, token);
         }
 
         public override async Task DeleteBucket(ILogger log, CancellationToken token)
         {
-            await _container.DeleteIfExistsAsync(token);
+            await _container.DeleteIfExistsAsync(cancellationToken: token);
         }
     }
 }