beam 1215 - upload to ECR

cli/cli/App.cs

@@ -4,6 +4,7 @@
 using Beamable.Common.Api.Auth;
 using Beamable.Common.Api.Realms;
 using Beamable.Common.Dependencies;
+using Beamable.Server;
 using cli.Commands.Project;
 using cli.Content;
 using cli.Dotnet;
@@ -20,6 +21,7 @@
 using System.CommandLine.Builder;
 using System.CommandLine.Invocation;
 using System.CommandLine.Parsing;
+using UnityEngine;
 
 namespace cli;
 
@@ -52,6 +54,7 @@ private static void ConfigureLogging()
 
 		BeamableLogProvider.Provider = new CliSerilogProvider();
 		CliSerilogProvider.LogContext.Value = Log.Logger;
+		Debug.Instance = new BeamableLoggerDebug();
 	}
 
 	/// <summary>

cli/cli/Commands/Profiling/RunNBomberCommand.cs

@@ -15,6 +15,9 @@ public class RunNBomberCommandArgs : CommandArgs
 
 	public bool includePrefix;
 	public int rps;
+	public int duration;
+	public string authHeader;
+	public bool includeAuth;
 }
 
 public class RunNBomberCommand : AppCommand<RunNBomberCommandArgs>
@@ -29,7 +32,10 @@ public override void Configure()
 		AddArgument(new Argument<string>("method", "The method name in the service to stress test"), (args, i) => args.method = i);
 		AddArgument(new Argument<string>("body", "The json body for each request"), (args, i) => args.jsonBody = i);
 		AddOption(new Option<bool>("--include-prefix", () => true, "If true, the generated .env file will include the local machine name as prefix"), (args, i) => args.includePrefix = i);
+		AddOption(new Option<bool>("--include-auth", () => true, "If true, the requests will be given the CLI's auth token"), (args, i) => args.includePrefix = i);
 		AddOption(new Option<int>("--rps", () => 50, "The requested requests per second for the test"), (args, i) => args.rps = i);
+		AddOption(new Option<int>("--duration", () => 30, "How long to run the test for"), (args, i) => args.duration = i);
+		AddOption(new Option<string>("--auth", "Include an auth header. This will override the --include-auth flag"), (args, i) => args.authHeader = i);
 	}
 
 	public override Task Handle(RunNBomberCommandArgs args)
@@ -50,6 +56,15 @@ public override Task Handle(RunNBomberCommandArgs args)
 						.WithHeader("X-DE-SCOPE", scope)
 						.WithBody(new StringContent(args.jsonBody));
 
+				if (args.includeAuth)
+				{
+					request = request.WithHeader("Authorization", $"Bearer {args.AppContext.Token.Token}");
+				}
+				if (!string.IsNullOrEmpty(args.authHeader))
+				{
+					request = request.WithHeader("Authorization", "Bearer 5d938ebe-9cbd-41d6-b077-d8dd89b99819");
+				}
+
 				// HttpCompletionOption: https://learn.microsoft.com/en-us/dotnet/api/system.net.http.httpcompletionoption?view=net-7.0
 
 				var clientArgs = new HttpClientArgs(
@@ -65,7 +80,7 @@ public override Task Handle(RunNBomberCommandArgs args)
 			.WithMaxFailCount(1)
 
 			.WithLoadSimulations(Simulation.Inject(rate: args.rps, interval: TimeSpan.FromSeconds(1),
-				during: TimeSpan.FromSeconds(30)))
+				during: TimeSpan.FromSeconds(args.duration)))
 		;
 
 		NBomberRunner

cli/cli/Commands/Services/ServicesDeployCommand.cs

@@ -15,6 +15,7 @@ public class ServicesDeployCommandArgs : LoginCommandArgs
 	public string FromJsonFile;
 	public string RemoteComment;
 	public string[] RemoteServiceComments;
+	public string registryUrl;
 }
 
 public class ServicesDeployCommand : AppCommand<ServicesDeployCommandArgs>
@@ -47,6 +48,8 @@ public override void Configure()
 
 		AddOption(new Option<string>("--comment", () => "", $"Requires --remote flag. Associates this comment along with the published Manifest. You'll be able to read it via the Beamable Portal"),
 			(args, i) => args.RemoteComment = i);
+		AddOption(new Option<string>("--registry-url", $"Requires --remote flag. Override the default registry upload url"),
+			(args, i) => args.registryUrl = i);
 
 		AddOption(new Option<string[]>("--service-comments", Array.Empty<string>, $"Requires --remote flag. Any number of 'BeamoId::Comment' strings. " +
 																				  $"\nAssociates each comment to the given Beamo Id if it's among the published services. You'll be able to read it via the Beamable Portal")
@@ -128,6 +131,10 @@ public override async Task Handle(ServicesDeployCommandArgs args)
 				Constants.PLATFORM_PRODUCTION => Constants.DOCKER_REGISTRY_PRODUCTION,
 				_ => throw new ArgumentOutOfRangeException()
 			};
+			if (!string.IsNullOrEmpty(args.registryUrl))
+			{
+				dockerRegistryUrl = args.registryUrl;
+			}
 
 			await AnsiConsole
 				.Progress()

cli/cli/Contants.cs

@@ -11,9 +11,9 @@ public static class Constants
 	public const string PLATFORM_PRODUCTION = "https://api.beamable.com";
 	public const string DEFAULT_PLATFORM = PLATFORM_PRODUCTION;
 
-	public const string DOCKER_REGISTRY_DEV = "https://dev-microservices.beamable.com/v2/";
-	public const string DOCKER_REGISTRY_STAGING = "https://staging-microservices.beamable.com/v2/";
-	public const string DOCKER_REGISTRY_PRODUCTION = "https://microservices.beamable.com/v2/";
+	public const string DOCKER_REGISTRY_DEV = "https://dev-ecr.beamable.com";
+	public const string DOCKER_REGISTRY_STAGING = "https://staging-ecr.beamable.com";
+	public const string DOCKER_REGISTRY_PRODUCTION = "https://ecr.beamable.com";
 
 	public const string CONFIG_CID = "cid";
 	public const string CONFIG_PID = "pid";

cli/cli/Services/BeamoLocalSystem_RemoteCommunication.cs

@@ -6,7 +6,9 @@
 
 using Beamable.Common;
 using Beamable.Serialization.SmallerJSON;
+using Beamable.Server;
 using ICSharpCode.SharpZipLib.Tar;
+using Serilog;
 using System.Net;
 using System.Net.Http.Headers;
 using System.Security.Cryptography;
@@ -204,22 +206,6 @@ public class ContainerUploadData
 		public long PartsAmount;
 	}
 
-	/// <summary>
-	/// Creates an <see cref="ContainerUploadData"/> structure for uploading a specific image to a specific customer, game and realm combination. 
-	/// </summary>
-	public ContainerUploadData PrepareContainerUploader(string cid, string gamePid, string realmPid, string token, string beamoId, string dockerRegistryUrl, BeamoServiceDefinition beamoService)
-	{
-		var containerUploadData = new ContainerUploadData { Md5 = MD5.Create(), Client = new HttpClient() };
-		containerUploadData.Client.DefaultRequestHeaders.Add("x-ks-clientid", cid);
-		containerUploadData.Client.DefaultRequestHeaders.Add("x-ks-projectid", realmPid);
-		containerUploadData.Client.DefaultRequestHeaders.Add("x-ks-token", token);
-		var serviceUniqueName = containerUploadData.Hash = GetHash($"{cid}_{gamePid}_{beamoId}", containerUploadData.Md5).Substring(0, 30);
-		containerUploadData.UploadBaseUri = $"{dockerRegistryUrl}{serviceUniqueName}";
-		containerUploadData.Sha256 = SHA256.Create();
-		containerUploadData.ServiceDefinition = beamoService;
-		return containerUploadData;
-	}
-
 	/// <summary>
 	/// Upload the specified container to the private Docker registry.
 	/// </summary>
@@ -247,8 +233,10 @@ public async Task UploadContainers(string[] beamoIds, string[] folders, string d
 					var tar = TarArchive.CreateInputTarArchive(stream, Encoding.Default);
 					tar.ExtractContents(folder);
 
-					var uploader = PrepareContainerUploader(cid, gamePid, realmPid, accessToken, sd.BeamoId, dockerRegistryUrl, sd);
-					await Upload(uploader, folder, cancellationToken, onContainerUploadProgress);
+					var service = new ECRUploaderService(cid, realmPid, gamePid, accessToken, sd.BeamoId, sd.TruncImageId, dockerRegistryUrl);
+					service.OnProgress += onContainerUploadProgress;
+					await service.Upload(folder, cancellationToken);
+
 					onContainerUploadCompleted?.Invoke(sd.BeamoId, true);
 				}
 				catch (Exception ex)
@@ -265,55 +253,7 @@ public async Task UploadContainers(string[] beamoIds, string[] folders, string d
 
 		await Task.WhenAll(uploadTasks);
 	}
-
-	/// <summary>
-	/// Upload a Docker image that has been expanded into a folder.
-	/// </summary>
-	/// <param name="folder">The filesystem path to the expanded image.</param>
-	private static async Task Upload(ContainerUploadData data, string folder, CancellationToken token, Action<string, float> onContainerUploadProgress = null)
-	{
-		token.ThrowIfCancellationRequested();
-		var manifest = DockerManifest.FromBytes(await File.ReadAllBytesAsync($"{folder}/manifest.json", token));
-		var uploadManifest = new Dictionary<string, object>
-		{
-			{ "schemaVersion", 2 }, { "mediaType", MEDIA_MANIFEST }, { "config", new Dictionary<string, object> { { "mediaType", MEDIA_CONFIG } } }, { "layers", new List<object>() },
-		};
-		var config = (Dictionary<string, object>)uploadManifest["config"];
-		var layers = (List<object>)uploadManifest["layers"];
-
-		// Upload the config JSON as a blob.
-		data.PartsAmount = manifest.layers.Length + 1;
-		data.PartsCompleted = 0;
-		onContainerUploadProgress?.Invoke(data.ServiceDefinition.BeamoId, (float)data.PartsCompleted / data.PartsAmount);
-
-		// Upload the file blob --- increase part and notify progress callback
-		var configResult = (await UploadFileBlob(data, $"{folder}/{manifest.config}", token));
-		token.ThrowIfCancellationRequested();
-		data.PartsCompleted += 1;
-		onContainerUploadProgress?.Invoke(data.ServiceDefinition.BeamoId, (float)data.PartsCompleted / data.PartsAmount);
-
-		config["digest"] = configResult.Digest;
-		config["size"] = configResult.Size;
-
-		// Upload all layer blobs.
-		var uploadIndexToJob = new SortedDictionary<int, Task<Dictionary<string, object>>>();
-		for (var i = 0; i < manifest.layers.Length; i++)
-		{
-			var layer = manifest.layers[i];
-			uploadIndexToJob.Add(i, UploadLayer(data, onContainerUploadProgress, $"{folder}/{layer}", token));
-		}
-
-		await Task.Run(() => Task.WhenAll(uploadIndexToJob.Values), token);
-		token.ThrowIfCancellationRequested();
-		foreach (var kvp in uploadIndexToJob)
-		{
-			layers.Add(kvp.Value.Result);
-		}
-
-		// Upload manifest JSON.
-		await UploadManifestJson(data, uploadManifest, data.ServiceDefinition.TruncImageId);
-	}
-
+
 	/// <summary>
 	/// Upload one layer of a Docker image, adding its digest to the upload
 	/// manifest when complete.

client/Packages/com.beamable.server/CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Runtime log level switching. In RealmConfig, use a key for service_logs|serviceName=logLevel.
 
+### Changed
+- Microservices upload to Beamable ECR instead of custom Docker registry.
 
 ## [1.14.0]
 ### Added

client/Packages/com.beamable.server/Editor/Uploader/ContainerUploadHarness.cs

@@ -91,7 +91,14 @@ public async Task UploadContainer(MicroserviceDescriptor descriptor, Cancellatio
 				var beamable = BeamEditorContext.Default;
 				await beamable.InitializePromise;
 
-				var uploader = new ContainerUploader(beamable, this, descriptor, imageId);
+				var uploader = new ECRUploaderService(
+					cid: beamable.Requester.Cid,
+					pid: beamable.Requester.Pid,
+					token: beamable.Requester.Token.Token,
+					productionPid: beamable.ProductionRealm.Pid,
+					serviceName: descriptor.Name,
+					imageId: imageId
+				);
 				await uploader.Upload(folder, token);
 
 				onSuccess?.Invoke();