Skip to content

Commit

Permalink
Replace Microsoft.Azure.Storage.Blob with Azure.Storage.Blobs pac…
Browse files Browse the repository at this point in the history 
…kage (#191)

* Use Azure.Storage.Blobs package

* Get tests compiling

* Clean-up usings

* fix nullref

* fix nits

* Fix nits suggested by rider
  • Loading branch information
@inthemedium
inthemedium authored Apr 20, 2024
1 parent d935a05 commit 433a0db
Show file tree
Hide file tree
Showing 14 changed files with 117 additions and 178 deletions.
2 changes: 1 addition & 1 deletion build/config.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<!-- Dependency versions -->
<PropertyGroup>
<NuGetPackageVersion>6.9.1</NuGetPackageVersion>
<MicrosoftAzureStorageBlobVersion>11.2.3</MicrosoftAzureStorageBlobVersion>
<AzureStorageBlobsVersion>12.19.1</AzureStorageBlobsVersion>
<JsonVersion>13.0.3</JsonVersion>
<CommandLineUtilsVersion>2.0.0</CommandLineUtilsVersion>
<NuGetTestHelpersVersion>2.1.36</NuGetTestHelpersVersion>
Expand Down
10 changes: 5 additions & 5 deletions doc/ci-server.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Github Action is free and common CI/CD tools to developer.

Now we take example to publish some nuget package in Github Action.

You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs and etc.
You can make it ok with other CI/CD tools like Jeknins, Gitlab Jobs, etc.

Now, we are going to publish nuget package to Azure by Github Action.

Expand Down Expand Up @@ -60,10 +60,10 @@ It means:

1. It will run CI action in ubuntu-latest OS
2. Step named `pack` will `cd` to the `src` directory and try to pack nuget packages versioned `1.0.0`
3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` diretocy to azure
3. Step named `Push nuget package to Azure storage` will publish packages placed in `pkgs` directory to azure
4. \${{secrets.SLEET_CONNECTIONSTRING}} is a secret, you can add it in you repository settings tab.

Put it togather as below:
Put it together as below:

```yml
name: Publish dev nuget package to azure
Expand Down Expand Up @@ -99,7 +99,7 @@ jobs:

### Some tips

#### Please pack nuget package to out it to specfic directory
#### Please pack nuget package to out it to specific directory

Maybe you publish nuget package via `dotnet` or `nuget` command as script below:

Expand All @@ -109,4 +109,4 @@ dotnet nuget push **/*.nupkg

Unfortunately, It is not support to publish package via `sleet push **/*.nupkg`.

So, please pack your nuget package to some directory like `pkg` as exmaple above.
So, please pack your nuget package to some directory like `pkg` as example above.
26 changes: 12 additions & 14 deletions src/SleetLib/FileSystem/AzureBlobLease.cs
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
using System;
using Azure.Storage.Blobs.Specialized;
using Azure.Storage.Blobs;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;

namespace Sleet
{
public class AzureBlobLease : IDisposable
{
private static readonly TimeSpan _leaseTime = new TimeSpan(0, 1, 0);
private readonly CloudBlockBlob _blob;
private readonly string _leaseId;
private readonly BlobClient _blob;

public AzureBlobLease(CloudBlockBlob blob)
public AzureBlobLease(BlobClient blob)
{
_blob = blob;
_leaseId = Guid.NewGuid().ToString();
LeaseId = Guid.NewGuid().ToString();
}

public string LeaseId => _leaseId;
public string LeaseId { get; }

/// <summary>
/// Makes a single attempt to get a lease.
Expand All @@ -30,21 +27,21 @@ public async Task<bool> GetLease()

try
{
actualLease = await _blob.AcquireLeaseAsync(_leaseTime, _leaseId);
actualLease = (await _blob.GetBlobLeaseClient(LeaseId).AcquireAsync(_leaseTime)).Value.LeaseId;
}
catch (Exception ex)
{
Debug.Fail($"GetLease failed: {ex}");
}

return StringComparer.Ordinal.Equals(_leaseId, actualLease);
return StringComparer.Ordinal.Equals(LeaseId, actualLease);
}

public async Task Renew()
{
try
{
await _blob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId));
await _blob.GetBlobLeaseClient(LeaseId).RenewAsync();
}
catch (Exception ex)
{
Expand All @@ -63,7 +60,8 @@ public void Release()
{
try
{
_blob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(_leaseId)).Wait(TimeSpan.FromSeconds(60));

_blob.GetBlobLeaseClient(LeaseId).ReleaseAsync().Wait(TimeSpan.FromSeconds(60));
}
catch (Exception ex)
{
Expand All @@ -72,4 +70,4 @@ public void Release()
}
}
}
}
}
57 changes: 26 additions & 31 deletions src/SleetLib/FileSystem/AzureFile.cs
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,36 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Storage.Blob;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs;
using NuGet.Common;
using System.IO.Compression;

namespace Sleet
{
public class AzureFile : FileBase
{
private readonly CloudBlockBlob _blob;
private readonly BlobClient _blob;

internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, CloudBlockBlob blob)
internal AzureFile(AzureFileSystem fileSystem, Uri rootPath, Uri displayPath, FileInfo localCacheFile, BlobClient blob)
: base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
{
_blob = blob;
}

protected override async Task CopyFromSource(ILogger log, CancellationToken token)
{
if (await _blob.ExistsAsync())
if (await _blob.ExistsAsync(token))
{
log.LogVerbose($"GET {_blob.Uri.AbsoluteUri}");

DeleteInternal();

using (var cache = File.OpenWrite(LocalCacheFile.FullName))
{
await _blob.DownloadToStreamAsync(cache);
await _blob.DownloadToAsync(cache, token);
}

// If the blob is compressed it needs to be decompressed locally before it can be used
if (_blob.Properties.ContentEncoding?.Equals("gzip", StringComparison.OrdinalIgnoreCase) == true)
var blobProperties = await _blob.GetPropertiesAsync(cancellationToken: token);
if (blobProperties.Value.ContentEncoding != null && blobProperties.Value.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase))
{
log.LogVerbose($"Decompressing {_blob.Uri.AbsoluteUri}");

Expand Down Expand Up @@ -60,71 +58,68 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
using (var cache = LocalCacheFile.OpenRead())
{
Stream writeStream = cache;
var blobHeaders = new BlobHttpHeaders
{
CacheControl = "no-store"
};

if (_blob.Uri.AbsoluteUri.EndsWith(".nupkg", StringComparison.Ordinal))
{
_blob.Properties.ContentType = "application/zip";
blobHeaders.ContentType = "application/zip";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".xml", StringComparison.Ordinal)
|| _blob.Uri.AbsoluteUri.EndsWith(".nuspec", StringComparison.Ordinal))
{
_blob.Properties.ContentType = "application/xml";
blobHeaders.ContentType = "application/xml";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".svg", StringComparison.Ordinal))
{
_blob.Properties.ContentType = "image/svg+xml";
blobHeaders.ContentType = "image/svg+xml";
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".json", StringComparison.Ordinal)
|| await JsonUtility.IsJsonAsync(LocalCacheFile.FullName))
{
_blob.Properties.ContentType = "application/json";
blobHeaders.ContentType = "application/json";

if (!SkipCompress())
{
_blob.Properties.ContentEncoding = "gzip";
blobHeaders.ContentEncoding = "gzip";
writeStream = await JsonUtility.GZipAndMinifyAsync(cache);
}
}
else if (_blob.Uri.AbsoluteUri.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)
|| _blob.Uri.AbsoluteUri.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase))
{
_blob.Properties.ContentType = "application/octet-stream";
blobHeaders.ContentType = "application/octet-stream";
}
else if (_blob.Uri.AbsoluteUri.EndsWith("/icon"))
{
_blob.Properties.ContentType = "image/png";
blobHeaders.ContentType = "image/png";
}
else
{
log.LogWarning($"Unknown file type: {_blob.Uri.AbsoluteUri}");
}

await _blob.UploadFromStreamAsync(writeStream);
await _blob.UploadAsync(writeStream, blobHeaders, cancellationToken: token);

writeStream.Dispose();
}

_blob.Properties.CacheControl = "no-store";

// TODO: re-enable this once it works again.
_blob.Properties.ContentMD5 = null;

await _blob.SetPropertiesAsync();
}
else if (await _blob.ExistsAsync())
else if (await _blob.ExistsAsync(token))
{
log.LogVerbose($"Removing {_blob.Uri.AbsoluteUri}");
await _blob.DeleteAsync();
await _blob.DeleteAsync(cancellationToken: token);
}
else
{
log.LogVerbose($"Skipping {_blob.Uri.AbsoluteUri}");
}
}

protected override Task<bool> RemoteExists(ILogger log, CancellationToken token)
protected override async Task<bool> RemoteExists(ILogger log, CancellationToken token)
{
return _blob.ExistsAsync();
return await _blob.ExistsAsync(token);
}
}
}
}
71 changes: 30 additions & 41 deletions src/SleetLib/FileSystem/AzureFileSystem.cs
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,24 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
using Azure.Storage.Blobs;
using NuGet.Common;
using Azure.Storage.Blobs.Models;

namespace Sleet
{
public class AzureFileSystem : FileSystemBase
{
public static readonly string AzureEmptyConnectionString = "DefaultEndpointsProtocol=https;AccountName=;AccountKey=;BlobEndpoint=";

private readonly CloudStorageAccount _azureAccount;
private readonly CloudBlobClient _client;
private readonly CloudBlobContainer _container;
private readonly BlobContainerClient _container;

public AzureFileSystem(LocalCache cache, Uri root, CloudStorageAccount azureAccount, string container)
: this(cache, root, root, azureAccount, container)
public AzureFileSystem(LocalCache cache, Uri root, BlobServiceClient blobServiceClient, string container)
: this(cache, root, root, blobServiceClient, container)
{
}

public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, CloudStorageAccount azureAccount, string container, string feedSubPath = null)
public AzureFileSystem(LocalCache cache, Uri root, Uri baseUri, BlobServiceClient blobServiceClient, string container, string feedSubPath = null)
: base(cache, root, baseUri, feedSubPath)
{
_azureAccount = azureAccount;
_client = _azureAccount.CreateCloudBlobClient();
_container = _client.GetContainerReference(container);
_container = blobServiceClient.GetBlobContainerClient(container);

var containerUri = UriUtility.EnsureTrailingSlash(_container.Uri);
var expectedPath = UriUtility.EnsureTrailingSlash(root);
Expand Down Expand Up @@ -63,14 +54,14 @@ public override ISleetFile Get(Uri path)
pair.Root,
pair.BaseURI,
LocalCache.GetNewTempPath(),
_container.GetBlockBlobReference(GetRelativePath(path))));
_container.GetBlobClient(GetRelativePath(path))));
}

public override async Task<bool> Validate(ILogger log, CancellationToken token)
{
log.LogInformation($"Verifying {_container.Uri.AbsoluteUri} exists.");

if (await _container.ExistsAsync())
if (await _container.ExistsAsync(token))
{
log.LogInformation($"Found {_container.Uri.AbsoluteUri}");
}
Expand All @@ -86,34 +77,32 @@ public override async Task<bool> Validate(ILogger log, CancellationToken token)
public override ISleetFileSystemLock CreateLock(ILogger log)
{
// Create blobs
var blob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
var messageBlob = _container.GetBlockBlobReference(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
var blob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFile)));
var messageBlob = _container.GetBlobClient(GetRelativePath(GetPath(AzureFileSystemLock.LockFileMessage)));
return new AzureFileSystemLock(blob, messageBlob, log);
}

public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, CancellationToken token)
{
string prefix = null;
var useFlatBlobListing = true;
var blobListingDetails = BlobListingDetails.All;
int? maxResults = null;
var results = _container.GetBlobsAsync();
var pages = results.AsPages();
var blobs = new List<ISleetFile>();

// Return all files except feedlock
var blobs = new List<IListBlobItem>();

BlobResultSegment result = null;
do
await foreach (var page in pages)
{
result = await _container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, result?.ContinuationToken, options: null, operationContext: null);
blobs.AddRange(result.Results);
// process page
blobs.AddRange(
page.Values
.Where(item => !item.Name.EndsWith(AzureFileSystemLock.LockFile, StringComparison.Ordinal))
.Where(item =>
string.IsNullOrEmpty(FeedSubPath) ||
item.Name.StartsWith(FeedSubPath, StringComparison.Ordinal))
.Select(item =>
Get(new BlobUriBuilder(_container.Uri) { BlobName = item.Name }.ToUri())
));
}
while (result.ContinuationToken != null);

// Skip the feed lock, and limit this to the current sub feed.
return blobs.Where(e => !e.Uri.AbsoluteUri.EndsWith($"/{AzureFileSystemLock.LockFile}"))
.Where(e => string.IsNullOrEmpty(FeedSubPath) || e.Uri.AbsoluteUri.StartsWith(UriUtility.EnsureTrailingSlash(BaseURI).AbsoluteUri, StringComparison.Ordinal))
.Select(e => Get(e.Uri))
.ToList();
return blobs;
}

public override string GetRelativePath(Uri uri)
Expand All @@ -128,19 +117,19 @@ public override string GetRelativePath(Uri uri)
return relativePath;
}

public override Task<bool> HasBucket(ILogger log, CancellationToken token)
public override async Task<bool> HasBucket(ILogger log, CancellationToken token)
{
return _container.ExistsAsync(token);
return await _container.ExistsAsync(token);
}

public override async Task CreateBucket(ILogger log, CancellationToken token)
{
await _container.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Blob, null, null, token);
await _container.CreateIfNotExistsAsync(PublicAccessType.BlobContainer, null, null, token);
}

public override async Task DeleteBucket(ILogger log, CancellationToken token)
{
await _container.DeleteIfExistsAsync(token);
await _container.DeleteIfExistsAsync(cancellationToken: token);
}
}
}
Loading

0 comments on commit 433a0db

Please sign in to comment.