beam 1215 - upload to ECR

.github/workflows/loadTestMicroservice.yml

@@ -47,7 +47,7 @@ jobs:
 
           echo "starting service"
           mkdir ../../../../logs
-          LOG_PATH=../../../../logs/serviceRuntime.log DOTNET_DiagnosticPorts=~/myport.sock ./standalone-microservice &
+          LOG_LEVEL=verbose LOG_PATH=../../../../logs/serviceRuntime.log DOTNET_DiagnosticPorts=~/myport.sock ./standalone-microservice &
 
           echo "starting profiler"
           cd ../../../..

cli/cli/App.cs

@@ -4,6 +4,7 @@
 using Beamable.Common.Api.Auth;
 using Beamable.Common.Api.Realms;
 using Beamable.Common.Dependencies;
+using Beamable.Server;
 using Beamable.Common.Semantics;
 using cli.Commands.Project;
 using cli.Content;
@@ -25,6 +26,7 @@
 using System.CommandLine.Builder;
 using System.CommandLine.Invocation;
 using System.CommandLine.Parsing;
+using UnityEngine;
 
 namespace cli;
 
@@ -59,6 +61,7 @@ private static void ConfigureLogging(Func<LoggerConfiguration, ILogger> configur
 
 		BeamableLogProvider.Provider = new CliSerilogProvider();
 		CliSerilogProvider.LogContext.Value = Log.Logger;
+		Debug.Instance = new BeamableLoggerDebug();
 	}
 
 	/// <summary>

cli/cli/CommandContext.cs

@@ -41,7 +41,7 @@ public static void SendResults<TChannel, TData>(this IResultSteam<TChannel, TDat
 		where TChannel : IResultChannel, new()
 	{
 		var channel = new TChannel(); // TODO: cache.
-		self.Reporter?.Report(channel.ChannelName, data);
+		self?.Reporter?.Report(channel.ChannelName, data);
 	}
 }
 

cli/cli/Commands/Profiling/RunNBomberCommand.cs

@@ -75,7 +75,6 @@ public override Task Handle(RunNBomberCommandArgs args)
 				{
 					request = request.WithHeader("Authorization", $"Bearer {args.authHeader}");
 				}
-
 				// HttpCompletionOption: https://learn.microsoft.com/en-us/dotnet/api/system.net.http.httpcompletionoption?view=net-7.0
 
 				var clientArgs = new HttpClientArgs(

cli/cli/Commands/Project/NewSolutionCommand.cs

@@ -87,6 +87,11 @@ public override async Task Handle(NewSolutionCommandArgs args)
 		// Default the solution name to the project name.
 		args.SolutionName = string.IsNullOrEmpty(args.SolutionName) ? args.ProjectName : args.SolutionName;
 
+		if (string.IsNullOrEmpty(args.directory))
+		{
+			args.directory = args.SolutionName;
+		}
+
 		// in the current directory, create a project using dotnet. 
 		var path = await args.ProjectService.CreateNewSolution(args);
 

cli/cli/Commands/Services/ServicesDeployCommand.cs

@@ -21,7 +21,7 @@ public class ServicesDeployCommandArgs : LoginCommandArgs
 
 	public string RemoteComment;
 	public string[] RemoteServiceComments;
-	public string dockerRegistryUrl;
+	public string registryUrl;
 }
 
 public class ServicesDeployCommand : AppCommand<ServicesDeployCommandArgs>,
@@ -55,14 +55,13 @@ public override void Configure()
 
 		AddOption(new Option<string>("--comment", () => "", $"Associates this comment along with the published Manifest. You'll be able to read it via the Beamable Portal"),
 			(args, i) => args.RemoteComment = i);
+		AddOption(new Option<string>("--registry-url", $"Requires --remote flag. Override the default registry upload url"),
+			(args, i) => args.registryUrl = i);
 
 		AddOption(new Option<string[]>("--service-comments", Array.Empty<string>, $"Any number of strings in the format BeamoId::Comment" +
 																				  $"\nAssociates each comment to the given Beamo Id if it's among the published services. You'll be able to read it via the Beamable Portal")
 		{ AllowMultipleArgumentsPerToken = true },
 			(args, i) => args.RemoteServiceComments = i);
-
-		AddOption(new Option<string>("--docker-registry-url", "A custom docker registry url to use when uploading. By default, the result from the beamo/registry network call will be used, " +
-															  "with minor string manipulation to add https scheme, remove port specificatino, and add /v2 "), (args, i) => args.dockerRegistryUrl = i);
 	}
 
 	public override async Task Handle(ServicesDeployCommandArgs args)
@@ -176,8 +175,8 @@ await _localBeamo.SynchronizeInstanceStatusWithDocker(_localBeamo.BeamoManifest,
 		}
 
 		// Get where we need to upload based on which platform env we are targeting
-		var dockerRegistryUrl = args.dockerRegistryUrl;
-		if (string.IsNullOrEmpty(dockerRegistryUrl))
+		var dockerRegistryUrl = args.registryUrl;
+		if (string.IsNullOrWhiteSpace(dockerRegistryUrl))
 		{
 			dockerRegistryUrl = await _remoteBeamo.GetDockerImageRegistryUri();
 		}

cli/cli/Contants.cs

@@ -29,6 +29,11 @@ public static class Constants
 	public const string PLATFORM_STAGING = "https://staging.api.beamable.com";
 	public const string PLATFORM_PRODUCTION = "https://api.beamable.com";
 	public const string DEFAULT_PLATFORM = PLATFORM_PRODUCTION;
+
+	public const string DOCKER_REGISTRY_DEV = "https://dev-ecr.beamable.com";
+	public const string DOCKER_REGISTRY_STAGING = "https://staging-ecr.beamable.com";
+	public const string DOCKER_REGISTRY_PRODUCTION = "https://ecr.beamable.com";
+
 	public const string CONFIG_CID = "cid";
 	public const string CONFIG_PID = "pid";
 	public const string CONFIG_DIR = "dir";

cli/cli/Services/BeamoLocalSystem_RemoteCommunication.cs

@@ -10,6 +10,9 @@
 using Beamable.Common.Content;
 using Beamable.Common.Reflection;
 using Beamable.Serialization.SmallerJSON;
+
+using Beamable.Server;
+using Serilog;
 using cli.Utils;
 using ICSharpCode.SharpZipLib.Tar;
 using Newtonsoft.Json.Linq;
@@ -311,8 +314,10 @@ public async Task UploadContainers(string[] beamoIds, string[] folders, string d
 					var tar = TarArchive.CreateInputTarArchive(stream, Encoding.Default);
 					tar.ExtractContents(folder);
 
-					var uploader = PrepareContainerUploader(cid, gamePid, realmPid, accessToken, sd.BeamoId, dockerRegistryUrl, sd);
-					await Upload(uploader, folder, cancellationToken, onContainerUploadProgress);
+					var service = new ECRUploaderService(cid, realmPid, gamePid, accessToken, sd.BeamoId, sd.TruncImageId, dockerRegistryUrl);
+					service.OnProgress += onContainerUploadProgress;
+					await service.Upload(folder, cancellationToken);
+
 					onContainerUploadCompleted?.Invoke(sd.BeamoId, true);
 				}
 				catch (Exception ex)
@@ -329,55 +334,7 @@ public async Task UploadContainers(string[] beamoIds, string[] folders, string d
 
 		await Task.WhenAll(uploadTasks);
 	}
-
-	/// <summary>
-	/// Upload a Docker image that has been expanded into a folder.
-	/// </summary>
-	/// <param name="folder">The filesystem path to the expanded image.</param>
-	private static async Task Upload(ContainerUploadData data, string folder, CancellationToken token, Action<string, float> onContainerUploadProgress = null)
-	{
-		token.ThrowIfCancellationRequested();
-		var manifest = DockerManifest.FromBytes(await File.ReadAllBytesAsync($"{folder}/manifest.json", token));
-		var uploadManifest = new Dictionary<string, object>
-		{
-			{ "schemaVersion", 2 }, { "mediaType", MEDIA_MANIFEST }, { "config", new Dictionary<string, object> { { "mediaType", MEDIA_CONFIG } } }, { "layers", new List<object>() },
-		};
-		var config = (Dictionary<string, object>)uploadManifest["config"];
-		var layers = (List<object>)uploadManifest["layers"];
-
-		// Upload the config JSON as a blob.
-		data.PartsAmount = manifest.layers.Length + 1;
-		data.PartsCompleted = 0;
-		onContainerUploadProgress?.Invoke(data.ServiceDefinition.BeamoId, (float)data.PartsCompleted / data.PartsAmount);
-
-		// Upload the file blob --- increase part and notify progress callback
-		var configResult = (await UploadFileBlob(data, $"{folder}/{manifest.config}", token));
-		token.ThrowIfCancellationRequested();
-		data.PartsCompleted += 1;
-		onContainerUploadProgress?.Invoke(data.ServiceDefinition.BeamoId, (float)data.PartsCompleted / data.PartsAmount);
-
-		config["digest"] = configResult.Digest;
-		config["size"] = configResult.Size;
-
-		// Upload all layer blobs.
-		var uploadIndexToJob = new SortedDictionary<int, Task<Dictionary<string, object>>>();
-		for (var i = 0; i < manifest.layers.Length; i++)
-		{
-			var layer = manifest.layers[i];
-			uploadIndexToJob.Add(i, UploadLayer(data, onContainerUploadProgress, $"{folder}/{layer}", token));
-		}
-
-		await Task.Run(() => Task.WhenAll(uploadIndexToJob.Values), token);
-		token.ThrowIfCancellationRequested();
-		foreach (var kvp in uploadIndexToJob)
-		{
-			layers.Add(kvp.Value.Result);
-		}
-
-		// Upload manifest JSON.
-		await UploadManifestJson(data, uploadManifest, data.ServiceDefinition.TruncImageId);
-	}
-
+
 	/// <summary>
 	/// Upload one layer of a Docker image, adding its digest to the upload
 	/// manifest when complete.

cli/cli/Services/ProjectService.cs

@@ -319,10 +319,6 @@ public Task<string> CreateNewSolution(NewSolutionCommandArgs args)
 	public async Task<string> CreateNewSolution(string directory, string solutionName, string projectName,
 		bool createCommonLibrary = true, string version = "", bool quiet = false)
 	{
-		if (string.IsNullOrEmpty(directory))
-		{
-			directory = solutionName;
-		}
 
 		string usedVersion = string.IsNullOrWhiteSpace(version) ? await GetVersion() : version;
 

client/Packages/com.beamable.server/CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Changed
+- Microservices upload to Beamable ECR instead of custom Docker registry.
+
 ## [1.19.7]
 
 no changes
@@ -160,7 +165,6 @@ no changes
 - Microservices can be started with an alias in the CID environment variable.
 
 ### Added
-
 - Runtime log level switching. In RealmConfig, use a key for `service_logs|serviceName=logLevel`.
 
 ## [1.14.0]

client/Packages/com.beamable.server/Editor/Uploader/ContainerUploadHarness.cs

@@ -91,7 +91,14 @@ public async Task UploadContainer(MicroserviceDescriptor descriptor, Cancellatio
 				var beamable = BeamEditorContext.Default;
 				await beamable.InitializePromise;
 
-				var uploader = new ContainerUploader(beamable, this, descriptor, imageId);
+				var uploader = new ECRUploaderService(
+					cid: beamable.Requester.Cid,
+					pid: beamable.Requester.Pid,
+					token: beamable.Requester.Token.Token,
+					productionPid: beamable.ProductionRealm.Pid,
+					serviceName: descriptor.Name,
+					imageId: imageId
+				);
 				await uploader.Upload(folder, token);
 
 				onSuccess?.Invoke();