Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replace Microsoft.Azure.Storage.Blob with Azure.Storage.Blobs package #191

Merged
merged 6 commits into from
Apr 20, 2024
Merged

Replace Microsoft.Azure.Storage.Blob with Azure.Storage.Blobs package #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
870d4ab
Use Azure.Storage.Blobs package
inthemedium Apr 18, 2024
6a9f681
Get tests compiling
inthemedium Apr 18, 2024
836e2bc
Clean-up usings
inthemedium Apr 18, 2024
2322575
fix nullref
inthemedium Apr 19, 2024
eb348a2
fix nits
inthemedium Apr 19, 2024
3c6399c
Fix nits suggested by rider
inthemedium Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/config.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<!-- Dependency versions -->
<PropertyGroup>
<NuGetPackageVersion>6.9.1</NuGetPackageVersion>
<MicrosoftAzureStorageBlobVersion>11.2.3</MicrosoftAzureStorageBlobVersion>
<AzureStorageBlobsVersion>12.19.1</AzureStorageBlobsVersion>
<JsonVersion>13.0.3</JsonVersion>
<CommandLineUtilsVersion>2.0.0</CommandLineUtilsVersion>
<NuGetTestHelpersVersion>2.1.36</NuGetTestHelpersVersion>
Expand Down
10 changes: 5 additions & 5 deletions doc/ci-server.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Github Action is free and common CI/CD tools to developer.

Now we take example to publish some nuget package in Github Action.

You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs and etc.
You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs, etc.

Now, we are going to publish nuget package to Azure by Github Action.

Expand Down Expand Up @@ -60,10 +60,10 @@ It means:

1. It will run CI action in ubuntu-latest OS
2. Step named `pack` will `cd` to the `src` directory and try to pack nuget packages versioned `1.0.0`
3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` diretocy to azure
3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` directory to azure
4. \${{secrets.SLEET_CONNECTIONSTRING}} is a secret, you can add it in you repository settings tab.

Put it togather as below:
Put it together as below:

```yml
name: Publish dev nuget package to azure
Expand Down Expand Up @@ -99,7 +99,7 @@ jobs:

### Some tips

#### Please pack nuget package to out it to specfic directory
#### Please pack nuget package to out it to specific directory

Maybe you publish nuget package via `dotnet` or `nuget` command as script below:

Expand All @@ -109,4 +109,4 @@ dotnet nuget push **/*.nupkg

Unfortunately, It is not support to publish package via `sleet push **/*.nupkg`.

So, please pack your nuget package to some directory like `pkg` as exmaple above.
So, please pack your nuget package to some directory like `pkg` as example above.
26 changes: 12 additions & 14 deletions src/SleetLib/FileSystem/AzureBlobLease.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
using System;
using Azure.Storage.Blobs.Specialized;
using Azure.Storage.Blobs;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;

namespace Sleet
{
public class AzureBlobLease : IDisposable
{
private static readonly TimeSpan _leaseTime = new TimeSpan(0, 1, 0);
private readonly CloudBlockBlob _blob;
private readonly string _leaseId;
private readonly BlobClient _blob;

public AzureBlobLease(CloudBlockBlob blob)
public AzureBlobLease(BlobClient blob)
{
_blob = blob;
_leaseId = Guid.NewGuid().ToString();
LeaseId = Guid.NewGuid().ToString();
}

public string LeaseId => _leaseId;
public string LeaseId { get; }

/// <summary>
/// Makes a single attempt to get a lease.
Expand All @@ -30,21 +27,21 @@ public async Task<bool> GetLease()

try
{
actualLease = await _blob.AcquireLeaseAsync(_leaseTime, _leaseId);
actualLease = (await _blob.GetBlobLeaseClient(LeaseId).AcquireAsync(_leaseTime)).Value.LeaseId;
}
catch (Exception ex)
{
Debug.Fail($"GetLease failed: {ex}");
}

return StringComparer.Ordinal.Equals(_leaseId, actualLease);
return StringComparer.Ordinal.Equals(LeaseId, actualLease);
}

public async Task Renew()
{
try
{
await _blob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId));
await _blob.GetBlobLeaseClient(LeaseId).RenewAsync();
}
catch (Exception ex)
{
Expand All @@ -63,7 +60,8 @@ public void Release()
{
try
{
_blob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId)).Wait(TimeSpan.FromSeconds(60));

_blob.GetBlobLeaseClient(LeaseId).ReleaseAsync().Wait(TimeSpan.FromSeconds(60));
}
catch (Exception ex)
{
Expand All @@ -72,4 +70,4 @@ public void Release()
}
}
}
}
}
57 changes: 26 additions & 31 deletions src/SleetLib/FileSystem/AzureFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,36 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Storage.Blob;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs;
using NuGet.Common;
using System.IO.Compression;

namespace Sleet
{
public class AzureFile : FileBase
{
private readonly CloudBlockBlob _blob;
private readonly BlobClient _blob;

internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, CloudBlockBlob blob)
internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, BlobClient blob)
: base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
{
_blob = blob;
}

protected override async Task CopyFromSource(ILogger log, CancellationToken token)
{
if (await _blob.ExistsAsync())
if (await _blob.ExistsAsync(token))
{
log.LogVerbose($"GET {_blob.Uri.AbsoluteUri}");

DeleteInternal();

using (var cache = File.OpenWrite(LocalCacheFile.FullName))
{
await _blob.DownloadToStreamAsync(cache);
await _blob.DownloadToAsync(cache, token);
}

// If the blob is compressed it needs to be decompressed locally before it can be used
if (_blob.Properties.ContentEncoding?.Equals("gzip", StringComparison.OrdinalIgnoreCase) == true)
var blobProperties = await _blob.GetPropertiesAsync(cancellationToken: token);
if (blobProperties.Value.ContentEncoding != null && blobProperties.Value.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase))
{
log.LogVerbose($"Decompressing {_blob.Uri.AbsoluteUri}");

Expand Down Expand Up @@ -60,71 +58,68 @@
using (var cache = LocalCacheFile.OpenRead())
{
Stream writeStream = cache;
var blobHeaders = new BlobHttpHeaders
{
CacheControl = "no-store"
};

if (_blob.Uri.AbsoluteUri.EndsWith(".nupkg", StringComparison.Ordinal))

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types

Check failure on line 66 in src/SleetLib/FileSystem/AzureFile.cs

View workflow job for this annotation

GitHub Actions / build-linux 

Type 'AzureFile' already defines a member called 'CopyFromSource' with the same parameter types
{
_blob.Properties.ContentType = "application/zip";
blobHeaders.ContentType = "application/zip";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".xml", StringComparison.Ordinal)
|| _blob.Uri.AbsoluteUri.EndsWith(".nuspec", StringComparison.Ordinal))
{
_blob.Properties.ContentType = "application/xml";
blobHeaders.ContentType = "application/xml";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".svg", StringComparison.Ordinal))
{
_blob.Properties.ContentType = "image/svg+xml";
blobHeaders.ContentType = "image/svg+xml";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".json", StringComparison.Ordinal)
|| await JsonUtility.IsJsonAsync(LocalCacheFile.FullName))
{
_blob.Properties.ContentType = "application/json";
blobHeaders.ContentType = "application/json";

if (!SkipCompress())
{
_blob.Properties.ContentEncoding = "gzip";
blobHeaders.ContentEncoding = "gzip";
writeStream = await JsonUtility.GZipAndMinifyAsync(cache);
}
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)
|| _blob.Uri.AbsoluteUri.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase))
{
_blob.Properties.ContentType = "application/octet-stream";
blobHeaders.ContentType = "application/octet-stream";
}
else if (_blob.Uri.AbsoluteUri.EndsWith("/icon"))
{
_blob.Properties.ContentType = "image/png";
blobHeaders.ContentType = "image/png";
}
else
{
log.LogWarning($"Unknown file type: {_blob.Uri.AbsoluteUri}");
}

await _blob.UploadFromStreamAsync(writeStream);
await _blob.UploadAsync(writeStream, blobHeaders, cancellationToken: token);

writeStream.Dispose();
}

_blob.Properties.CacheControl = "no-store";

// TODO: re-enable this once it works again.
_blob.Properties.ContentMD5 = null;

await _blob.SetPropertiesAsync();
}
else if (await _blob.ExistsAsync())
else if (await _blob.ExistsAsync(token))
{
log.LogVerbose($"Removing {_blob.Uri.AbsoluteUri}");
await _blob.DeleteAsync();
await _blob.DeleteAsync(cancellationToken: token);
}
else
{
log.LogVerbose($"Skipping {_blob.Uri.AbsoluteUri}");
}
}

protected override Task<bool> RemoteExists(ILogger log, CancellationToken token)
protected override async Task<bool> RemoteExists(ILogger log, CancellationToken token)
{
return _blob.ExistsAsync();
return await _blob.ExistsAsync(token);
}
}
}
}
71 changes: 30 additions & 41 deletions src/SleetLib/FileSystem/AzureFileSystem.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,7 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
Expand Up @@ -8,26 +1,24 @@
using Azure.Storage.Blobs;
using NuGet.Common;
using Azure.Storage.Blobs.Models;

namespace Sleet
{
public class AzureFileSystem : FileSystemBase
{
public static readonly string AzureEmptyConnectionString = "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;BlobEndpoint=";

private readonly CloudStorageAccount _azureAccount;
private readonly CloudBlobClient _client;
private readonly CloudBlobContainer _container;
private readonly BlobContainerClient _container;

public AzureFileSystem(LocalCache cache, Uri root, CloudStorageAccount azureAccount, string container)
: this(cache, root, root, azureAccount, container)
public AzureFileSystem(LocalCache cache, Uri root, BlobServiceClient blobServiceClient, string container)
: this(cache, root, root, blobServiceClient, container)
{
}

public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, CloudStorageAccount azureAccount, string container, string feedSubPath = null)
public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, BlobServiceClient blobServiceClient, string container, string feedSubPath = null)
: base(cache, root, baseUri, feedSubPath)
{
_azureAccount = azureAccount;
_client = _azureAccount.CreateCloudBlobClient();
_container = _client.GetContainerReference(container);
_container = blobServiceClient.GetBlobContainerClient(container);

var containerUri = UriUtility.EnsureTrailingSlash(_container.Uri);
var expectedPath = UriUtility.EnsureTrailingSlash(root);
Expand Down Expand Up @@ -63,14 +54,14 @@
pair.Root,
pair.BaseURI,
LocalCache.GetNewTempPath(),
_container.GetBlockBlobReference(GetRelativePath(path))));
_container.GetBlobClient(GetRelativePath(path))));
}

public override async Task<bool> Validate(ILogger log, CancellationToken token)
{
log.LogInformation($"Verifying {_container.Uri.AbsoluteUri} exists.");

if (await _container.ExistsAsync())
if (await _container.ExistsAsync(token))
{
log.LogInformation($"Found {_container.Uri.AbsoluteUri}");
}
Expand All @@ -86,34 +77,32 @@
public override ISleetFileSystemLock CreateLock(ILogger log)
{
// Create blobs
var blob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
var messageBlob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
var blob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
var messageBlob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
return new AzureFileSystemLock(blob, messageBlob, log);
}

public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, CancellationToken token)
{
string prefix = null;
var useFlatBlobListing = true;
var blobListingDetails = BlobListingDetails.All;
Copy link
Contributor Author

@inthemedium inthemedium Apr 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There isn't an 1:1 equivalent of this enum in the new API, so I kept the default values which looks like this:

        /// <summary>
        /// The <see cref="GetBlobsAsync"/> operation returns an async
        /// sequence of blobs in this container.  Enumerating the blobs may
        /// make multiple requests to the service while fetching all the
        /// values.  Blobs are ordered lexicographically by name.
        ///
        /// For more information, see
        /// <see href="https://docs.microsoft.com/rest/api/storageservices/list-blobs">
        /// List Blobs</see>.
        /// </summary>
        /// <param name="traits">
        /// Specifies trait options for shaping the blobs.
        /// </param>
        /// <param name="states">
        /// Specifies state options for filtering the blobs.
        /// </param>
        /// <param name="prefix">
        /// Specifies a string that filters the results to return only blobs
        /// whose name begins with the specified <paramref name="prefix"/>.
        /// </param>
        /// <param name="cancellationToken">
        /// Optional <see cref="CancellationToken"/> to propagate
        /// notifications that the operation should be cancelled.
        /// </param>
        /// <returns>
        /// An <see cref="AsyncPageable{T}"/> describing the
        /// blobs in the container.
        /// </returns>
        /// <remarks>
        /// A <see cref="RequestFailedException"/> will be thrown if
        /// a failure occurs.
        /// </remarks>
        public virtual AsyncPageable<BlobItem> GetBlobsAsync(
            BlobTraits traits = BlobTraits.None,
            BlobStates states = BlobStates.None,
            string prefix = default,
            CancellationToken cancellationToken = default)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

docs for states and traits:
https://learn.microsoft.com/en-us/dotnet/api/azure.storage.blobs.models.blobstates?view=azure-dotnet
https://learn.microsoft.com/en-us/dotnet/api/azure.storage.blobs.models.blobtraits?view=azure-dotnet

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For states the default looks correct.
Possibly the metadata trait is needed to get the properties such as the content type and content encoding, I can't tell from the docs if those come down by default.

int? maxResults = null;
var results = _container.GetBlobsAsync();
var pages = results.AsPages();
var blobs = new List<ISleetFile>();

// Return all files except feedlock
var blobs = new List<IListBlobItem>();

BlobResultSegment result = null;
do
await foreach (var page in pages)
{
result = await _container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, result?.ContinuationToken, options: null, operationContext: null);
blobs.AddRange(result.Results);
// process page
blobs.AddRange(
page.Values
.Where(item => !item.Name.EndsWith(AzureFileSystemLock.LockFile, StringComparison.Ordinal))
.Where(item =>
string.IsNullOrEmpty(FeedSubPath) ||
item.Name.StartsWith(FeedSubPath, StringComparison.Ordinal))
.Select(item =>
Get(new BlobUriBuilder(_container.Uri) { BlobName = item.Name }.ToUri())
));
}
while (result.ContinuationToken != null);

// Skip the feed lock, and limit this to the current sub feed.
return blobs.Where(e => !e.Uri.AbsoluteUri.EndsWith($"/{AzureFileSystemLock.LockFile}"))
.Where(e => string.IsNullOrEmpty(FeedSubPath) || e.Uri.AbsoluteUri.StartsWith(UriUtility.EnsureTrailingSlash(BaseURI).AbsoluteUri, StringComparison.Ordinal))
.Select(e => Get(e.Uri))
.ToList();
return blobs;
}

public override string GetRelativePath(Uri uri)
Expand All @@ -128,19 +117,19 @@
return relativePath;
}

public override Task<bool> HasBucket(ILogger log, CancellationToken token)
public override async Task<bool> HasBucket(ILogger log, CancellationToken token)
{
return _container.ExistsAsync(token);
return await _container.ExistsAsync(token);
}

public override async Task CreateBucket(ILogger log, CancellationToken token)
{
await _container.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Blob, null, null, token);
await _container.CreateIfNotExistsAsync(PublicAccessType.BlobContainer, null, null, token);
}

public override async Task DeleteBucket(ILogger log, CancellationToken token)
{
await _container.DeleteIfExistsAsync(token);
await _container.DeleteIfExistsAsync(cancellationToken: token);
}
}
}
Loading
Loading