Skip to content

Commit

Permalink
[XABT] Use ZipArchive to build APKs. (#9623)
Browse files Browse the repository at this point in the history 
It would appear we no longer require features not available in `System.IO.Compression.ZipArchive` in order to build `.apk`/`.aab` files.  Additionally, it appears that `ZipArchive` is noticeably faster than our current usage of `LibZipSharp`.  Switch the `BuildArchive` task to preferring `ZipArchive` over `LibZipSharp` when possible for building APKs.  The `LibZipSharp` implementation is maintained as a fallback and in case future new `.apk`/`.aab` requirements necessitate its use.

Archive sizes for `.apk` increase 1.3%-2.7% which seems acceptable. `.aab` sizes remain roughly the same, likely because `bundletool` repacks them.

### Implementation Notes

- `netstandard2.0` does not expose API for examining a `CRC` or `CompressionMethod` of an existing entry in a Zip file, though both `net472` and `net9.0` have private fields.  As such, we use reflection to access these private fields.  If the runtime we are using does not have these fields, we will fall back to using `LibZipSharp` as we do now.
- Abstract our required Zip API to `IZipArchive` so that we can switch between `System.IO.Compression.ZipFile` and `LibZipSharp` as needed.
- Due to a bug on .NET Framework where uncompressed files are stored as `Deflate` with a compression level of `0` instead of being stored as `Store`, if we detect that we need to store uncompressed files we will fall back to `LibZipSharp`.  This seems to be an uncommon scenario that is not hit by any of our default flows.
- Can force fallback to `LibZipSharp` with `$(_AndroidUseLibZipSharp)`=`true`.

### Performance

Measurements of the `BuildArchive` task when using the `android` template for initial and incremental build scenarios.

#### Debug - FastDev

| Scenario        | `main`    | This PR |
| --------------- | ------- | ------- |
| Full            | 2.428 s | 339 ms  |
| NoChanges       | not run | not run |
| ChangeResource  | 34 ms   | 19 ms   |
| AddResource     | 23 ms   | 17 ms   |
| ChangeCSharp    | not run | not run |
| ChangeCSharpJLO | not run | not run |
| Archive Size | 5,390,140 bytes | 5,537,596 bytes |

#### Debug - EmbedAssembliesInApk

| Scenario        | `main`     | This PR |
| --------------- | -------- | ------- |
| Full            | 34.856 s | 4.313 s |
| NoChanges       | not run  | not run |
| ChangeResource  | 33.385 s | 4.165 s |
| AddResource     | 32.206 s | 3.963 s |
| ChangeCSharp    | 32.060 s | 3.979 s |
| ChangeCSharpJLO | 33.161 s | 3.997 s |
| Archive Size | 76,653,152 bytes | 77,710,097 bytes |

#### Release

| Scenario        | `main`    | This PR |
| --------------- | ------- | ------- |
| Full            | 2.195 s | 387 ms  |
| NoChanges       | not run | not run |
| ChangeResource  | 134 ms  | 73 ms   |
| AddResource     | 685 ms  | 182 ms  |
| ChangeCSharp    | 705 ms  | 142 ms  |
| ChangeCSharpJLO | 703 ms  | 149 ms  |
| Archive Size | 6,917,153 bytes | 6,917,319 bytes |


CI build that falls back to `LibZipSharp` to ensure it still passes our tests: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=10720142
  • Loading branch information
@jpobst
jpobst authored Jan 24, 2025
1 parent f7260a7 commit 7454deb
Show file tree
Hide file tree
Showing 6 changed files with 468 additions and 107 deletions.
124 changes: 86 additions & 38 deletions src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Microsoft.Android.Build.Tasks;
using Microsoft.Build.Framework;
using Xamarin.Tools.Zip;
Expand Down Expand Up @@ -32,6 +33,8 @@ public class BuildArchive : AndroidTask

public string? UncompressedFileExtensions { get; set; }

public bool UseLibZipSharp { get; set; }

public string? ZipFlushFilesLimit { get; set; }

public string? ZipFlushSizeLimit { get; set; }
Expand All @@ -43,8 +46,10 @@ public class BuildArchive : AndroidTask

public override bool RunTask ()
{
var is_aab = string.Compare (AndroidPackageFormat, "aab", true) == 0;

// Nothing needs to be compressed with app bundles. BundleConfig.json specifies the final compression mode.
if (string.Compare (AndroidPackageFormat, "aab", true) == 0)
if (is_aab)
uncompressedMethod = CompressionMethod.Default;

var refresh = true;
Expand All @@ -57,7 +62,7 @@ public override bool RunTask ()
refresh = false;
}

using var apk = new ZipArchiveEx (ApkOutputPath, FileMode.Open);
using var apk = ZipArchiveDotNet.Create (Log, ApkOutputPath, System.IO.Compression.ZipArchiveMode.Update, ShouldFallbackToLibZipSharp ());

// Set up AutoFlush
if (int.TryParse (ZipFlushFilesLimit, out int flushFilesLimit)) {
Expand All @@ -73,10 +78,9 @@ public override bool RunTask ()
var existingEntries = new List<string> ();

if (refresh) {
for (var i = 0; i < apk.Archive.EntryCount; i++) {
var entry = apk.Archive.ReadEntry ((ulong) i);
Log.LogDebugMessage ($"Registering item {entry.FullName}");
existingEntries.Add (entry.FullName);
foreach (var entry in apk.GetAllEntryNames ()) {
Log.LogDebugMessage ($"Registering item {entry}");
existingEntries.Add (entry);
}
}

Expand All @@ -98,6 +102,11 @@ public override bool RunTask ()
Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`");
}

if (entryName == "AndroidManifest.xml" && is_aab) {
Log.LogDebugMessage ("Renaming AndroidManifest.xml to manifest/AndroidManifest.xml");
entryName = "manifest/AndroidManifest.xml";
}

Log.LogDebugMessage ($"Deregistering item {entryName}");
existingEntries.Remove (entryName);

Expand All @@ -106,24 +115,28 @@ public override bool RunTask ()
continue;
}

if (apk.Archive.ContainsEntry (entryName)) {
ZipEntry e = apk.Archive.ReadEntry (entryName);
if (apk.ContainsEntry (entryName)) {
var e = apk.GetEntry (entryName);
// check the CRC values as the ModifiedDate is always 01/01/1980 in the aapt generated file.
if (entry.CRC == e.CRC && entry.CompressedSize == e.CompressedSize) {
Log.LogDebugMessage ($"Skipping {entryName} from {ApkInputPath} as its up to date.");
continue;
}

// Delete the existing entry so we can replace it with the new one.
apk.DeleteEntry (entryName);
}

var ms = new MemoryStream ();
entry.Extract (ms);
ms.Position = 0;
Log.LogDebugMessage ($"Refreshing {entryName} from {ApkInputPath}");
apk.Archive.AddStream (ms, entryName, compressionMethod: entry.CompressionMethod);
apk.AddEntry (ms, entryName, entry.CompressionMethod.ToCompressionLevel ());
}
}
}

apk.FixupWindowsPathSeparators ((a, b) => Log.LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`"));
apk.FixupWindowsPathSeparators (Log);

// Add the files to the apk
foreach (var file in FilesToAddToArchive) {
Expand All @@ -135,6 +148,8 @@ public override bool RunTask ()
return !Log.HasLoggedErrors;
}

apk_path = apk_path.Replace ('\\', '/');

// This is a temporary hack for adding files directly from inside a .jar/.aar
// into the APK. Eventually another task should be writing them to disk and just
// passing us a filename like everything else.
Expand All @@ -145,7 +160,7 @@ public override bool RunTask ()
// eg: "obj/myjar.jar#myfile.txt"
var jar_file_path = disk_path.Substring (0, disk_path.Length - (jar_entry_name.Length + 1));

if (apk.Archive.Any (ze => ze.FullName == apk_path)) {
if (apk.ContainsEntry (apk_path)) {
Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", jar_entry_name, Path.GetFileName (jar_file_path));
continue;
}
Expand All @@ -165,7 +180,7 @@ public override bool RunTask ()
}

Log.LogDebugMessage ($"Adding {jar_entry_name} from {jar_file_path} as the archive file is out of date.");
apk.AddEntryAndFlush (data, apk_path);
apk.AddEntry (data, apk_path);
}

continue;
Expand All @@ -180,63 +195,96 @@ public override bool RunTask ()
if (string.Compare (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0)
continue;

Log.LogDebugMessage ($"Removing {entry} as it is not longer required.");
apk.Archive.DeleteEntry (entry);
Log.LogDebugMessage ($"Removing {entry} as it is no longer required.");
apk.DeleteEntry (entry);
}

if (string.Compare (AndroidPackageFormat, "aab", true) == 0)
if (is_aab)
FixupArchive (apk);

return !Log.HasLoggedErrors;
}

bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePath, ITaskItem item, List<string> existingEntries)
// .NET Framework has a bug where it doesn't handle uncompressed files correctly.
// It writes them as "compressed" (DEFLATE) but with a compression level of 0. This causes
// issues with Android, which expect uncompressed files to be stored correctly.
// We can work around this by using LibZipSharp, which doesn't have this bug.
// This is only necessary if we're on .NET Framework (MSBuild in VSWin) and we have uncompressed files.
bool ShouldFallbackToLibZipSharp ()
{
var compressionMethod = GetCompressionMethod (item);
existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/'));
// Explicitly requested via MSBuild property.
if (UseLibZipSharp) {
Log.LogDebugMessage ("Falling back to LibZipSharp because '$(_AndroidUseLibZipSharp)' is 'true'.");
return true;
}

if (apk.SkipExistingFile (file, inArchivePath, compressionMethod)) {
Log.LogDebugMessage ($"Skipping {file} as the archive file is up to date.");
// .NET 6+ handles uncompressed files correctly, so we don't need to fallback.
if (RuntimeInformation.FrameworkDescription == ".NET") {
Log.LogDebugMessage ("Using System.IO.Compression because we're running on .NET 6+.");
return false;
}

Log.LogDebugMessage ($"Adding {file} as the archive file is out of date.");
apk.AddFileAndFlush (file, inArchivePath, compressionMethod);
// Nothing is going to get written uncompressed, so we don't need to fallback.
if (uncompressedMethod != CompressionMethod.Store) {
Log.LogDebugMessage ("Using System.IO.Compression because uncompressedMethod isn't 'Store'.");
return false;
}

return true;
// No uncompressed file extensions were specified, so we don't need to fallback.
if (UncompressedFileExtensionsSet.Count == 0) {
Log.LogDebugMessage ("Using System.IO.Compression because no uncompressed file extensions were specified.");
return false;
}

// See if any of the files to be added need to be uncompressed.
foreach (var file in FilesToAddToArchive) {
var file_path = file.ItemSpec;

// Handle files from inside a .jar/.aar
if (file.GetMetadataOrDefault ("JavaArchiveEntry", (string?)null) is string jar_entry_name)
file_path = jar_entry_name;

if (UncompressedFileExtensionsSet.Contains (Path.GetExtension (file_path))) {
Log.LogDebugMessage ($"Falling back to LibZipSharp because '{file_path}' needs to be stored uncompressed.");
return true;
}
}

Log.LogDebugMessage ("Using System.IO.Compression because no files need to be stored uncompressed.");
return false;
}

bool AddFileToArchiveIfNewer (IZipArchive apk, string file, string inArchivePath, ITaskItem item, List<string> existingEntries)
{
var compressionMethod = GetCompressionLevel (item);
existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/'));

return apk.AddFileIfChanged (Log, file, inArchivePath, compressionMethod);
}

/// <summary>
/// aapt2 is putting AndroidManifest.xml in the root of the archive instead of at manifest/AndroidManifest.xml that bundletool expects.
/// I see no way to change this behavior, so we can move the file for now:
/// https://github.com/aosp-mirror/platform_frameworks_base/blob/e80b45506501815061b079dcb10bf87443bd385d/tools/aapt2/LoadedApk.h#L34
/// </summary>
void FixupArchive (ZipArchiveEx zip)
void FixupArchive (IZipArchive zip)
{
if (!zip.Archive.ContainsEntry ("AndroidManifest.xml")) {
if (!zip.ContainsEntry ("AndroidManifest.xml")) {
Log.LogDebugMessage ($"No AndroidManifest.xml. Skipping Fixup");
return;
}

var entry = zip.Archive.ReadEntry ("AndroidManifest.xml");
Log.LogDebugMessage ($"Fixing up AndroidManifest.xml to be manifest/AndroidManifest.xml.");

if (zip.Archive.ContainsEntry ("manifest/AndroidManifest.xml"))
zip.Archive.DeleteEntry (zip.Archive.ReadEntry ("manifest/AndroidManifest.xml"));
if (zip.ContainsEntry ("manifest/AndroidManifest.xml"))
zip.DeleteEntry ("manifest/AndroidManifest.xml");

entry.Rename ("manifest/AndroidManifest.xml");
zip.MoveEntry ("AndroidManifest.xml", "manifest/AndroidManifest.xml");
}

CompressionMethod GetCompressionMethod (ITaskItem item)
System.IO.Compression.CompressionLevel GetCompressionLevel (ITaskItem item)
{
var compression = item.GetMetadataOrDefault ("Compression", "");

if (compression.HasValue ()) {
if (Enum.TryParse (compression, out CompressionMethod result))
return result;
}

return UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default;
return (UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default).ToCompressionLevel ();
}

HashSet<string> ParseUncompressedFileExtensions ()
Expand Down
46 changes: 46 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/UtilityExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
using System;
using System.IO;
using System.IO.Compression;
using Xamarin.Tools.Zip;

namespace Xamarin.Android.Tasks;

static class UtilityExtensions
{
public static System.IO.Compression.CompressionLevel ToCompressionLevel (this CompressionMethod method)
{
switch (method) {
case CompressionMethod.Store:
return System.IO.Compression.CompressionLevel.NoCompression;
case CompressionMethod.Default:
case CompressionMethod.Deflate:
return System.IO.Compression.CompressionLevel.Optimal;
default:
throw new ArgumentOutOfRangeException (nameof (method), method, null);
}
}

public static CompressionMethod ToCompressionMethod (this System.IO.Compression.CompressionLevel level)
{
switch (level) {
case System.IO.Compression.CompressionLevel.NoCompression:
return CompressionMethod.Store;
case System.IO.Compression.CompressionLevel.Optimal:
return CompressionMethod.Deflate;
default:
throw new ArgumentOutOfRangeException (nameof (level), level, null);
}
}

public static FileMode ToFileMode (this ZipArchiveMode mode)
{
switch (mode) {
case ZipArchiveMode.Create:
return FileMode.Create;
case ZipArchiveMode.Update:
return FileMode.Open;
default:
throw new ArgumentOutOfRangeException (nameof (mode), mode, null);
}
}
}
Loading

0 comments on commit 7454deb

Please sign in to comment.