From bbabe2a7e12c00a8a23a8ae02fe0d3c7aa217d74 Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Thu, 31 Aug 2023 02:53:57 +0700 Subject: [PATCH] Fix IConfiguration adapter (#365) * Fix IConfiguration adapter case sensitivity * Add normalization flag and update documentation * Update documentation * Update API Approval list * Fix typo * Fix unit test --- README.md | 118 +++++++++++- .../CoreApiSpec.ApproveCore.verified.txt | 4 +- .../ConfigurationHoconAdapterTest.cs | 182 +++++++++++++++--- src/Akka.Hosting/AkkaHostingExtensions.cs | 6 +- .../ConfigurationHoconAdapter.cs | 22 +-- 5 files changed, 285 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 526652cd..f41c1552 100644 --- a/README.md +++ b/README.md @@ -312,8 +312,9 @@ The `dependencyResolver.Props()` call will leverage the `Act ## IConfiguration To HOCON Adapter The `AddHocon` extension method can convert `Microsoft.Extensions.Configuration` `IConfiguration` into HOCON `Config` instance and adds it to the ActorSystem being configured. -* All variable name are automatically converted to lower case. -* All "." (period) in the `IConfiguration` key will be treated as a HOCON object key separator +* Unlike `IConfiguration`, all HOCON key names are **case sensitive**. +* **Unless enclosed inside double quotes**, all "." (period) in the `IConfiguration` key will be treated as a HOCON object key separator +* `IConfiguration` **does not support object composition**, if you declare the same key multiple times inside multiple configuration providers (JSON/environment variables/etc), **only the last one declared will take effect**. * For environment variable configuration provider: * "__" (double underline) will be converted to "." (period). * "_" (single underline) will be converted to "-" (dash). @@ -340,16 +341,33 @@ __Example:__ } } } - ``` +``` Environment variables: ```powershell -AKKA__ACTOR__TELEMETRY__ENABLED=true +AKKA__ACTOR__TELEMETRY__ENABLE=true AKKA__CLUSTER__SEED_NODES__0=akka.tcp//mySystem@localhost:4055 AKKA__CLUSTER__SEED_NODES__1=akka.tcp//mySystem@localhost:4056 AKKA__CLUSTER__SEED_NODE_TIMEOUT=00:00:05 - ``` +``` + +Note the integer parseable key inside the seed-nodes configuration, seed-nodes will be parsed as an array. These environment variables will be parsed as HOCON settings: + +```hocon +akka { + actor { + telemetry.enabled: on + } + cluster { + seed-nodes: [ + "akka.tcp//mySystem@localhost:4055", + "akka.tcp//mySystem@localhost:4056" + ] + seed-node-timeout: 5s + } +} +``` Example code: @@ -392,6 +410,96 @@ var host = new HostBuilder() }); ``` +### Special Characters And Case Sensitivity + +This advanced usage of the `IConfiguration` adapter is solely used for edge cases where HOCON key capitalization needs to be preserved, such as declaring serialization binding. Note that when you're using this feature, none of the keys are normalized, you will have to write all of your keys in a HOCON compatible way. + +`appsettings.json`: + +```json +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "akka": { + "\"Key.With.Dots\"": "Key Value", + "cluster": { + "roles": [ "front-end", "back-end" ], + "min-nr-of-members": 3, + "log-info": true + } + } +} +``` + +Note that "Key.With.Dots" needs to be inside escaped double quotes, this is a HOCON requirement that preserves the "." (period) inside HOCON property names. + +Environment variables: + +```powershell +PS C:/> [Environment]::SetEnvironmentVariable('akka__actor__telemetry__enabled', 'true') +PS C:/> [Environment]::SetEnvironmentVariable('akka__actor__serialization_bindings__"System.Object"', 'hyperion') +PS C:/> [Environment]::SetEnvironmentVariable('akka__cluster__seed_nodes__0', 'akka.tcp//mySystem@localhost:4055') +PS C:/> [Environment]::SetEnvironmentVariable('akka__cluster__seed_nodes__1', 'akka.tcp//mySystem@localhost:4056') +PS C:/> [Environment]::SetEnvironmentVariable('akka__cluster__seed_node_timeout', '00:00:05') +``` + +Note that: +1. All of the environment variable names are in lower case, except "System.Object" where it needs to preserve name capitalization. +2. To set serialization binding via environment variable, you have to use "." (period) instead of "__" (double underscore), this might be problematic for some shell scripts and there is no way of getting around this. + +Example code: + +```csharp +/* +Both appsettings.json and environment variables are combined +into HOCON configuration: + +akka { + "Key.With.Dots": Key Value + actor { + telemetry.enabled: on + serialization-bindings { + "System.Object" = hyperion + } + } + cluster { + roles: [ "front-end", "back-end" ] + seed-nodes: [ + "akka.tcp//mySystem@localhost:4055", + "akka.tcp//mySystem@localhost:4056" + ] + min-nr-of-members: 3 + seed-node-timeout: 5s + log-info: true + } +} +*/ +var host = new HostBuilder() + .ConfigureHostConfiguration(builder => + { + // Setup IConfiguration to load from appsettings.json and + // environment variables + builder + .AddJsonFile("appsettings.json") + .AddEnvironmentVariables(); + }) + .ConfigureServices((context, services) => + { + services.AddAkka("mySystem", (builder, provider) => + { + // convert IConfiguration to HOCON + var akkaConfig = context.Configuration.GetSection("akka"); + // Note the last method argument is set to false + builder.AddHocon(akkaConfig, HoconAddMode.Prepend, false); + }); + }); +``` + [Back to top](#akkahosting) diff --git a/src/Akka.Hosting.API.Tests/verify/CoreApiSpec.ApproveCore.verified.txt b/src/Akka.Hosting.API.Tests/verify/CoreApiSpec.ApproveCore.verified.txt index 7b7be39b..4dadffb1 100644 --- a/src/Akka.Hosting.API.Tests/verify/CoreApiSpec.ApproveCore.verified.txt +++ b/src/Akka.Hosting.API.Tests/verify/CoreApiSpec.ApproveCore.verified.txt @@ -66,7 +66,7 @@ namespace Akka.Hosting public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddAkka(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, string actorSystemName, System.Action builder) where T : Akka.Hosting.AkkaHostedService { } public static Akka.Hosting.AkkaConfigurationBuilder AddHocon(this Akka.Hosting.AkkaConfigurationBuilder builder, Akka.Configuration.Config hocon, Akka.Hosting.HoconAddMode addMode) { } - public static Akka.Hosting.AkkaConfigurationBuilder AddHocon(this Akka.Hosting.AkkaConfigurationBuilder builder, Microsoft.Extensions.Configuration.IConfiguration configuration, Akka.Hosting.HoconAddMode addMode) { } + public static Akka.Hosting.AkkaConfigurationBuilder AddHocon(this Akka.Hosting.AkkaConfigurationBuilder builder, Microsoft.Extensions.Configuration.IConfiguration configuration, Akka.Hosting.HoconAddMode addMode, bool normalizeKeys = true) { } public static Akka.Hosting.AkkaConfigurationBuilder AddHoconFile(this Akka.Hosting.AkkaConfigurationBuilder builder, string hoconFilePath, Akka.Hosting.HoconAddMode addMode) { } public static Akka.Hosting.AkkaConfigurationBuilder AddSetup(this Akka.Hosting.AkkaConfigurationBuilder builder, Akka.Actor.Setup.Setup setup) { } public static Akka.Hosting.AkkaConfigurationBuilder WithActorAskTimeout(this Akka.Hosting.AkkaConfigurationBuilder builder, System.TimeSpan timeout) { } @@ -187,7 +187,7 @@ namespace Akka.Hosting.Configuration { public static class ConfigurationHoconAdapter { - public static Akka.Configuration.Config ToHocon(this Microsoft.Extensions.Configuration.IConfiguration config) { } + public static Akka.Configuration.Config ToHocon(this Microsoft.Extensions.Configuration.IConfiguration config, bool normalizeKeys = true) { } } } namespace Akka.Hosting.Coordination diff --git a/src/Akka.Hosting.Tests/Configuration/ConfigurationHoconAdapterTest.cs b/src/Akka.Hosting.Tests/Configuration/ConfigurationHoconAdapterTest.cs index a21450a5..a8e31c9f 100644 --- a/src/Akka.Hosting.Tests/Configuration/ConfigurationHoconAdapterTest.cs +++ b/src/Akka.Hosting.Tests/Configuration/ConfigurationHoconAdapterTest.cs @@ -1,6 +1,5 @@ // ----------------------------------------------------------------------- // -// Copyright (C) 2009-2022 Lightbend Inc. // Copyright (C) 2013-2022 .NET Foundation // // ----------------------------------------------------------------------- @@ -8,7 +7,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using System.Threading.Tasks; using Akka.Configuration; using Akka.Hosting.Configuration; using FluentAssertions; @@ -19,10 +20,13 @@ namespace Akka.Hosting.Tests.Configuration; -public class ConfigurationHoconAdapterTest +public class ConfigurationHoconAdapterTest: IAsyncLifetime { private const string ConfigSource = @"{ ""akka"": { + ""actor.serialization-bindings"" : { + ""\""System.Int32\"""": ""json"" + }, ""cluster"": { ""roles"": [ ""front-end"", ""back-end"" ], ""role"" : { @@ -44,43 +48,166 @@ public class ConfigurationHoconAdapterTest ""test4"": 4 }"; - private readonly Config _config; + private readonly IConfigurationRoot _root; public ConfigurationHoconAdapterTest() { - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_1__A", "A VALUE"); - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_1__B", "B VALUE"); - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_1__C__D", "D"); - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_2__0", "ZERO"); - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_2__22", "TWO"); - Environment.SetEnvironmentVariable("AKKA__TEST_VALUE_2__1", "ONE"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__A", "A VALUE"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__B", "B VALUE"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__C__D", "D"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__0", "ZERO"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__22", "TWO"); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__1", "ONE"); + + Environment.SetEnvironmentVariable("akka__test_value_3__a", "a value"); + Environment.SetEnvironmentVariable("akka__test_value_3__b", "b value"); + Environment.SetEnvironmentVariable("akka__test_value_3__c__d", "d"); + Environment.SetEnvironmentVariable("akka__test_value_4__0", "zero"); + Environment.SetEnvironmentVariable("akka__test_value_4__22", "two"); + Environment.SetEnvironmentVariable("akka__test_value_4__1", "one"); + Environment.SetEnvironmentVariable("akka__actor__serialization_bindings2__\"System.Object\"", "hyperion"); using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ConfigSource)); - var configuration = new ConfigurationBuilder() + _root = new ConfigurationBuilder() .AddJsonStream(stream) .AddEnvironmentVariables() .Build(); - _config = configuration.ToHocon(); } + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__A", null); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__B", null); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_1__C__D", null); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__0", null); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__22", null); + Environment.SetEnvironmentVariable("akka__TEST_VALUE_2__1", null); + + Environment.SetEnvironmentVariable("akka__test_value_3__a", null); + Environment.SetEnvironmentVariable("akka__test_value_3__b", null); + Environment.SetEnvironmentVariable("akka__test_value_3__c__d", null); + Environment.SetEnvironmentVariable("akka__test_value_4__0", null); + Environment.SetEnvironmentVariable("akka__test_value_4__22", null); + Environment.SetEnvironmentVariable("akka__test_value_4__1", null); + Environment.SetEnvironmentVariable("akka__actor__serialization_bindings2__\"System.Object\"", null); + + return Task.CompletedTask; + } + #region Adapter unit tests - [Fact(DisplayName = "Adaptor should read environment variable sourced configuration correctly")] + [Fact(DisplayName = "Normalized adaptor should read environment variable sourced configuration correctly")] public void EnvironmentVariableTest() { - _config.GetString("akka.test-value-1.a").Should().Be("A VALUE"); - _config.GetString("akka.test-value-1.b").Should().Be("B VALUE"); - _config.GetString("akka.test-value-1.c.d").Should().Be("D"); - var array = _config.GetStringList("akka.test-value-2"); + var config = _root.ToHocon(); + // should be normalized + config.GetString("akka.test-value-1.a").Should().Be("A VALUE"); + config.GetString("akka.TEST-VALUE-1.A").Should().BeNull(); + + // should be normalized + config.GetString("akka.test-value-1.b").Should().Be("B VALUE"); + config.GetString("akka.TEST-VALUE-1.B").Should().BeNull(); + + // should be normalized + config.GetString("akka.test-value-1.c.d").Should().Be("D"); + config.GetString("akka.TEST-VALUE-1.C.D").Should().BeNull(); + + // should be normalized + var array = config.GetStringList("akka.test-value-2"); array[0].Should().Be("ZERO"); array[1].Should().Be("ONE"); array[2].Should().Be("TWO"); + config.GetStringList("AKKA.TEST-VALUE-2").Should().BeEmpty(); + + // proper cased environment vars should read just fine + config.GetString("akka.test-value-3.a").Should().Be("a value"); + config.GetString("akka.test-value-3.b").Should().Be("b value"); + config.GetString("akka.test-value-3.c.d").Should().Be("d"); + array = config.GetStringList("akka.test-value-4"); + array[0].Should().Be("zero"); + array[1].Should().Be("one"); + array[2].Should().Be("two"); + + // edge case should also be normalized and not usable + var bindings = config.GetConfig("akka.actor.serialization-bindings2").AsEnumerable() + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + bindings.ContainsKey("System.Object").Should().BeFalse(); + bindings.ContainsKey("system.object").Should().BeTrue(); + bindings["system.object"].GetString().Should().Be("hyperion"); + } + + [Fact(DisplayName = "Non-normalized adaptor should read environment variable sourced configuration correctly")] + public void EnvironmentVariableCaseSensitiveTest() + { + var config = _root.ToHocon(false); + + // should not be normalized + config.GetString("akka.TEST-VALUE-1.A").Should().Be("A VALUE"); + config.GetString("akka.test-value-1.a").Should().BeNull(); + + // should not be normalized + config.GetString("akka.TEST-VALUE-1.B").Should().Be("B VALUE"); + config.GetString("akka.test-value-1.b").Should().BeNull(); + + // should not be normalized + config.GetString("akka.TEST-VALUE-1.C.D").Should().Be("D"); + config.GetString("akka.test-value-1.c.d").Should().BeNull(); + + // should not be normalized + config.GetStringList("akka.test-value-2").Should().BeEmpty(); + var array = config.GetStringList("akka.TEST-VALUE-2"); + array[0].Should().Be("ZERO"); + array[1].Should().Be("ONE"); + array[2].Should().Be("TWO"); + + // proper cased environment vars should read just fine + config.GetString("akka.test-value-3.a").Should().Be("a value"); + config.GetString("akka.test-value-3.b").Should().Be("b value"); + config.GetString("akka.test-value-3.c.d").Should().Be("d"); + array = config.GetStringList("akka.test-value-4"); + array[0].Should().Be("zero"); + array[1].Should().Be("one"); + array[2].Should().Be("two"); + + // edge case should not be normalized and usable + var bindings = config.GetConfig("akka.actor.serialization-bindings2").AsEnumerable() + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + bindings.ContainsKey("System.Object").Should().BeTrue(); + bindings["System.Object"].GetString().Should().Be("hyperion"); + } + + [Fact(DisplayName = "Non-normalized Adaptor should read quote enclosed key inside JSON settings correctly")] + public void NonNormalizedJsonQuotedKeyTest() + { + var config = _root.ToHocon(false); + var bindings = config.GetConfig("akka.actor.serialization-bindings").AsEnumerable() + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + bindings.ContainsKey("System.Int32").Should().BeTrue(); + bindings["System.Int32"].GetString().Should().Be("json"); + } + + [Fact(DisplayName = "Normalized Adaptor should read quote enclosed key inside JSON settings incorrectly")] + public void NormalizedJsonQuotedKeyTest() + { + var config = _root.ToHocon(); + var bindings = config.GetConfig("akka.actor.serialization-bindings").AsEnumerable() + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + + bindings.ContainsKey("System.Int32").Should().BeFalse(); + bindings.ContainsKey("system.int32").Should().BeTrue(); + bindings["system.int32"].GetString().Should().Be("json"); } [Fact(DisplayName = "Adaptor should expand keys")] public void EncodedKeyTest() { - var test2 = _config.GetConfig("test2"); + var config = _root.ToHocon(); + var test2 = config.GetConfig("test2"); test2.Should().NotBeNull(); test2.GetBoolean("a").Should().BeTrue(); test2.GetTimeSpan("b.c").Should().Be(2.Seconds()); @@ -91,18 +218,19 @@ public void EncodedKeyTest() [Fact(DisplayName = "Adaptor should convert correctly")] public void ArrayTest() { - _config.GetString("test1").Should().Be("test1 content"); - _config.GetInt("test3").Should().Be(3); - _config.GetInt("test4").Should().Be(4); + var config = _root.ToHocon(); + config.GetString("test1").Should().Be("test1 content"); + config.GetInt("test3").Should().Be(3); + config.GetInt("test4").Should().Be(4); - _config.GetStringList("akka.cluster.roles").Should().BeEquivalentTo("front-end", "back-end"); - _config.GetInt("akka.cluster.role.back-end").Should().Be(5); - _config.GetString("akka.cluster.app-version").Should().Be("1.0.0"); - _config.GetInt("akka.cluster.min-nr-of-members").Should().Be(99); - _config.GetStringList("akka.cluster.seed-nodes").Should() + config.GetStringList("akka.cluster.roles").Should().BeEquivalentTo("front-end", "back-end"); + config.GetInt("akka.cluster.role.back-end").Should().Be(5); + config.GetString("akka.cluster.app-version").Should().Be("1.0.0"); + config.GetInt("akka.cluster.min-nr-of-members").Should().Be(99); + config.GetStringList("akka.cluster.seed-nodes").Should() .BeEquivalentTo("akka.tcp://system@somewhere.com:9999"); - _config.GetBoolean("akka.cluster.log-info").Should().BeFalse(); - _config.GetBoolean("akka.cluster.log-info-verbose").Should().BeTrue(); + config.GetBoolean("akka.cluster.log-info").Should().BeFalse(); + config.GetBoolean("akka.cluster.log-info-verbose").Should().BeTrue(); } #endregion diff --git a/src/Akka.Hosting/AkkaHostingExtensions.cs b/src/Akka.Hosting/AkkaHostingExtensions.cs index e2e08f0a..abf42fda 100644 --- a/src/Akka.Hosting/AkkaHostingExtensions.cs +++ b/src/Akka.Hosting/AkkaHostingExtensions.cs @@ -153,13 +153,15 @@ public static AkkaConfigurationBuilder AddHocon(this AkkaConfigurationBuilder bu /// The builder instance being configured. /// The instance to be converted to HOCON . /// The - defaults to appending this HOCON as a fallback. + /// /// The same instance originally passed in. public static AkkaConfigurationBuilder AddHocon( this AkkaConfigurationBuilder builder, IConfiguration configuration, - HoconAddMode addMode) + HoconAddMode addMode, + bool normalizeKeys = true) { - return builder.AddHoconConfiguration(configuration.ToHocon(), addMode); + return builder.AddHoconConfiguration(configuration.ToHocon(normalizeKeys), addMode); } /// diff --git a/src/Akka.Hosting/Configuration/ConfigurationHoconAdapter.cs b/src/Akka.Hosting/Configuration/ConfigurationHoconAdapter.cs index 87ecc48d..37992d11 100644 --- a/src/Akka.Hosting/Configuration/ConfigurationHoconAdapter.cs +++ b/src/Akka.Hosting/Configuration/ConfigurationHoconAdapter.cs @@ -15,20 +15,20 @@ namespace Akka.Hosting.Configuration { public static class ConfigurationHoconAdapter { - public static Config ToHocon(this IConfiguration config) + public static Config ToHocon(this IConfiguration config, bool normalizeKeys = true) { var rootObject = new HoconObject(); if (config is IConfigurationSection section) { - var value = section.ExpandKey(rootObject); - value.AppendValue(section.ToHoconElement()); + var value = section.ExpandKey(rootObject, normalizeKeys); + value.AppendValue(section.ToHoconElement(normalizeKeys)); } else { foreach (var child in config.GetChildren()) { - var value = child.ExpandKey(rootObject); - value.AppendValue(child.ToHoconElement()); + var value = child.ExpandKey(rootObject, normalizeKeys); + value.AppendValue(child.ToHoconElement(normalizeKeys)); } } @@ -37,11 +37,11 @@ public static Config ToHocon(this IConfiguration config) return new Config(new HoconRoot(rootValue)); } - private static HoconValue ExpandKey(this IConfigurationSection config, HoconObject parent) + private static HoconValue ExpandKey(this IConfigurationSection config, HoconObject parent, bool normalizeKeys) { // Sanitize configuration brought in from environment variables, // "__" are already converted to ":" by the environment configuration provider. - var sanitized = config.Key.ToLowerInvariant().Replace("_", "-"); + var sanitized = (normalizeKeys ? config.Key.ToLowerInvariant() : config.Key).Replace("_", "-"); var keys = sanitized.SplitDottedPathHonouringQuotes().ToList(); // No need to expand the chain @@ -69,7 +69,7 @@ private static HoconValue ExpandKey(this IConfigurationSection config, HoconObje return currentObj.GetOrCreateKey(keys[0]); } - private static IHoconElement ToHoconElement(this IConfigurationSection config) + private static IHoconElement ToHoconElement(this IConfigurationSection config, bool normalizeKeys) { if (config.IsArray()) { @@ -77,7 +77,7 @@ private static IHoconElement ToHoconElement(this IConfigurationSection config) foreach (var child in config.GetChildren().OrderBy(c => int.Parse(c.Key))) { var value = new HoconValue(); - var element = child.ToHoconElement(); + var element = child.ToHoconElement(normalizeKeys); value.AppendValue(element); array.Add(value); } @@ -89,8 +89,8 @@ private static IHoconElement ToHoconElement(this IConfigurationSection config) var rootObject = new HoconObject(); foreach (var child in config.GetChildren()) { - var value = child.ExpandKey(rootObject); - value.AppendValue(child.ToHoconElement()); + var value = child.ExpandKey(rootObject, normalizeKeys); + value.AppendValue(child.ToHoconElement(normalizeKeys)); } return rootObject; }