POC: Allow trust to be changed for a running tentacle without the need to restart

source/Octopus.Manager.Tentacle/TentacleConfiguration/InstanceConfigurationLoader.cs

@@ -97,7 +97,6 @@ public SimpleApplicationInstanceSelectorForTentacleManager(
                 {
                     Current = LoadInstance();
                     canLoadCurrentInstance = true;
-
                 }
                 catch
                 {
@@ -112,7 +111,7 @@ public SimpleApplicationInstanceSelectorForTentacleManager(
             ApplicationInstanceConfiguration LoadInstance()
             {
                 var (aggregatedKeyValueStore, writableConfig) = LoadConfigurationStore(configFilePath);
-                return new ApplicationInstanceConfiguration(null, configFilePath, aggregatedKeyValueStore, writableConfig);
+                return new ApplicationInstanceConfiguration(null, configFilePath, aggregatedKeyValueStore, writableConfig, null);
             }
 
             (IKeyValueStore, IWritableKeyValueStore) LoadConfigurationStore(string configurationPath)

source/Octopus.Tentacle.Tests.Integration/TrustConfigurationTests.cs

@@ -0,0 +1,104 @@
+#nullable enable
+using System;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Threading.Tasks;
+using FluentAssertions;
+using Halibut;
+using NUnit.Framework;
+using Octopus.Tentacle.Certificates;
+using Octopus.Tentacle.Client;
+using Octopus.Tentacle.Client.Retries;
+using Octopus.Tentacle.Client.Scripts;
+using Octopus.Tentacle.Contracts.Legacy;
+using Octopus.Tentacle.Contracts.Observability;
+using Octopus.Tentacle.Diagnostics;
+using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
+using Octopus.Tentacle.Tests.Integration.Util.Builders;
+
+namespace Octopus.Tentacle.Tests.Integration
+{
+    [IntegrationTestTimeout]
+    public class TrustConfigurationTests : IntegrationTest
+    {
+        [Test]
+        [TentacleConfigurations(testPolling: false)]
+        public async Task ChangingTheTrustedThumbprintsForAListeningTentacleShouldNotRequireARestart(TentacleConfigurationTestCase tentacleConfigurationTestCase)
+        {
+            await using (var clientAndTentacle = await tentacleConfigurationTestCase
+                             .CreateBuilder()
+                             .WithRetryDuration(TimeSpan.FromSeconds(20))
+                             .Build(CancellationToken))
+            {
+                using var newCertificate = new CertificateGenerator(new SystemLog()).GenerateNew($"cn={Guid.NewGuid()}");
+
+                // Add a new trusted thumbprint
+                var addTrustCommand = new TestExecuteShellScriptCommandBuilder()                    
+                    .SetScriptBodyForCurrentOs(
+$@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}""
+.\Tentacle.exe configure --instance {clientAndTentacle.RunningTentacle.InstanceName} --trust {newCertificate.Thumbprint}",
+$@"#!/bin/sh
+cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}""
+./Tentacle configure --instance {clientAndTentacle.RunningTentacle.InstanceName} --trust {newCertificate.Thumbprint}")
+                    .Build();
+
+
+                var result = await clientAndTentacle.TentacleClient.ExecuteScript(addTrustCommand, CancellationToken);
+                result.LogExecuteScriptOutput(Logger);
+                result.ScriptExecutionResult.ExitCode.Should().Be(0);
+                result.ProcessOutput.Any(x => x.Text.Contains("Adding 1 trusted Octopus Servers")).Should().BeTrue("Adding 1 trusted Octopus Servers should be logged");
+
+                // Ensure the new thumbprint is trusted
+                var checkCommunicationCommand = new TestExecuteShellScriptCommandBuilder()                    
+                    .SetScriptBody(new ScriptBuilder().Print("Success..."))
+                    .Build();
+
+                var tentacleClientUsingNewCertificate = BuildTentacleClientForNewCertificate(newCertificate, clientAndTentacle);
+
+                result = await tentacleClientUsingNewCertificate.ExecuteScript(checkCommunicationCommand, CancellationToken);
+                result.LogExecuteScriptOutput(Logger);
+                result.ScriptExecutionResult.ExitCode.Should().Be(0);
+                result.ProcessOutput.Any(x => x.Text.Contains("Success...")).Should().BeTrue("Success... should be logged");;
+
+                // Remove trust for the old thumbprint
+                var removeTrustCommand = new TestExecuteShellScriptCommandBuilder()                    
+                    .SetScriptBodyForCurrentOs(
+                        $@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}""
+.\Tentacle.exe configure --instance {clientAndTentacle.RunningTentacle.InstanceName} --remove-trust {clientAndTentacle.Server.Thumbprint}",
+                        $@"#!/bin/sh
+cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}""
+./Tentacle configure --instance {clientAndTentacle.RunningTentacle.InstanceName} --remove-trust {clientAndTentacle.Server.Thumbprint}")
+                    .Build();
+
+                result = await tentacleClientUsingNewCertificate.ExecuteScript(removeTrustCommand, CancellationToken);
+                result.LogExecuteScriptOutput(Logger);
+                result.ScriptExecutionResult.ExitCode.Should().Be(0);
+                result.ProcessOutput.Any(x => x.Text.Contains("Removing 1 trusted Octopus Servers")).Should().BeTrue("Removing 1 trusted Octopus Servers should be logged");
+
+                // Ensure the old thumbprint is no longer trusted
+                await AssertionExtensions
+                    .Should(async () => await clientAndTentacle.TentacleClient.ExecuteScript(checkCommunicationCommand, CancellationToken))
+                    .ThrowAsync<HalibutClientException>();
+            }
+        }
+
+        static TentacleClient BuildTentacleClientForNewCertificate(X509Certificate2 newCertificate, ClientAndTentacle clientAndTentacle)
+        {
+            var halibutRuntime = new HalibutRuntimeBuilder()
+                .WithServerCertificate(newCertificate)
+                .WithHalibutTimeoutsAndLimits(clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits)
+                .WithLegacyContractSupport()
+                .Build();
+
+            TentacleClient.CacheServiceWasNotFoundResponseMessages(halibutRuntime);
+
+            var retrySettings = new RpcRetrySettings(true, TimeSpan.FromSeconds(5));
+            var clientOptions = new TentacleClientOptions(retrySettings);
+
+            var tentacleClient = new TentacleClient(clientAndTentacle.ServiceEndPoint, halibutRuntime, new DefaultScriptObserverBackoffStrategy(), new NoTentacleClientObserver(), clientOptions);
+
+            return tentacleClient;
+        }
+    }
+}

source/Octopus.Tentacle.Tests/Commands/ConfigureCommandFixture.cs

@@ -29,7 +29,7 @@ public override void SetUp()
             fileSystem = Substitute.For<IOctopusFileSystem>();
             log = Substitute.For<ISystemLog>();
             var selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
             Command = new ConfigureCommand(new Lazy<IWritableTentacleConfiguration>(() => tentacleConfiguration), new Lazy<IWritableHomeConfiguration>(() => new StubHomeConfiguration()), fileSystem, log, selector, Substitute.For<ILogFileOnlyLogger>());
         }
 

source/Octopus.Tentacle.Tests/Commands/DeregisterMachineCommandFixture.cs

@@ -34,7 +34,7 @@ public override void SetUp()
             proxyConfig = Substitute.For<IProxyConfigParser>();
             log = Substitute.For<ISystemLog>();
             selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(new ApplicationInstanceConfiguration("my-instance", "myconfig.config", Substitute.For<IKeyValueStore>(), Substitute.For<IWritableKeyValueStore>()));
+            selector.Current.Returns(new ApplicationInstanceConfiguration("my-instance", "myconfig.config", Substitute.For<IKeyValueStore>(), Substitute.For<IWritableKeyValueStore>(), null));
         }
 
         [Test]

source/Octopus.Tentacle.Tests/Commands/DeregisterWorkerCommandFixture.cs

@@ -34,7 +34,7 @@ public override void SetUp()
             proxyConfig = Substitute.For<IProxyConfigParser>();
             log = Substitute.For<ISystemLog>();
             selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(new ApplicationInstanceConfiguration("my-instance", "myconfig.config", Substitute.For<IKeyValueStore>(), Substitute.For<IWritableKeyValueStore>()));
+            selector.Current.Returns(new ApplicationInstanceConfiguration("my-instance", "myconfig.config", Substitute.For<IKeyValueStore>(), Substitute.For<IWritableKeyValueStore>(), null));
         }
 
         [Test]

source/Octopus.Tentacle.Tests/Commands/ImportCertificateCommandFixture.cs

@@ -23,7 +23,7 @@ public void BeforeEachTest()
         {
             configuration = Substitute.For<IWritableTentacleConfiguration>();
             var selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(_ => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            selector.Current.Returns(_ => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
             Command = new ImportCertificateCommand(
                 new Lazy<IWritableTentacleConfiguration>(() => configuration),
                 Substitute.For<ISystemLog>(),

source/Octopus.Tentacle.Tests/Commands/NewCertificateCommandFixture.cs

@@ -24,7 +24,7 @@ public override void SetUp()
             log = Substitute.For<ISystemLog>();
             configuration = new StubTentacleConfiguration();
             var selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
             Command = new NewCertificateCommand(new Lazy<IWritableTentacleConfiguration>(() => configuration), log, selector, new Lazy<ICertificateGenerator>(() => Substitute.For<ICertificateGenerator>()), Substitute.For<ILogFileOnlyLogger>());
         }
 

source/Octopus.Tentacle.Tests/Commands/ProxyConfigurationCommandFixture.cs

@@ -27,7 +27,7 @@ public void SetupForEachTest()
             configFile = $"{homeDirectory}\\File.config";
 
             applicationInstanceSelector = Substitute.For<IApplicationInstanceSelector>();
-            applicationInstanceSelector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            applicationInstanceSelector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
 
             octopusFileSystem.AppendToFile(configFile,
                 "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +

source/Octopus.Tentacle.Tests/Commands/RegisterMachineCommandFixture.cs

@@ -59,7 +59,7 @@ public void BeforeEachTest()
             octopusClientInitializer.CreateClient(Arg.Any<ApiEndpointOptions>(), false)
                 .Returns(Task.FromResult(octopusAsyncClient));
             var selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
             Command = new RegisterMachineCommand(new Lazy<IRegisterMachineOperation>(() => operation),
                                                  new Lazy<IWritableTentacleConfiguration>(() => configuration),
                                                  log,

source/Octopus.Tentacle.Tests/Commands/RegisterWorkerCommandFixture.cs

@@ -60,7 +60,7 @@ public void BeforeEachTest()
                 .Returns(Task.FromResult(octopusAsyncClient));
 
             var applicationInstanceSelector = Substitute.For<IApplicationInstanceSelector>();
-            applicationInstanceSelector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            applicationInstanceSelector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
 
             Command = new RegisterWorkerCommand(new Lazy<IRegisterWorkerOperation>(() => operation),
                 new Lazy<IWritableTentacleConfiguration>(() => configuration),

source/Octopus.Tentacle.Tests/Commands/RunAgentCommandFixture.cs

@@ -62,7 +62,7 @@ public override void SetUp()
                 Substitute.For<ILogFileOnlyLogger>(),
                 backgroundTasks.Select(bt => new Lazy<IBackgroundTask>(() => bt)).ToList());
 
-            selector.Current.Returns(new ApplicationInstanceConfiguration("MyTentacle", null, null, null));
+            selector.Current.Returns(new ApplicationInstanceConfiguration("MyTentacle", null, null, null, null));
         }
 
         [Test]

source/Octopus.Tentacle.Tests/Commands/ServerCommsCommandTest.cs

@@ -27,7 +27,7 @@ public void SetUp()
         {
             configuration = new StubTentacleConfiguration();
             var selector = Substitute.For<IApplicationInstanceSelector>();
-            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!));
+            selector.Current.Returns(info => new ApplicationInstanceConfiguration(null, null!, null!, null!, null));
             command = new ServerCommsCommand(
                 new Lazy<IWritableTentacleConfiguration>(() => configuration),
                 new InMemoryLog(),

source/Octopus.Tentacle.Tests/Commands/StubTentacleConfiguration.cs

@@ -39,10 +39,13 @@ public IEnumerable<string> TrustedOctopusThumbprints
         public IProxyConfiguration ProxyConfiguration { get; set; } = null!;
         public IPollingProxyConfiguration PollingProxyConfiguration { get; set; } = null!;
         public bool IsRegistered { get; set; } = false;
+        public ConfigurationChangedEventHandler? Changed { get; set; }
+
         public void WriteTo(IWritableKeyValueStore outputStore, IEnumerable<string> excluding)
         {
             throw new NotImplementedException();
         }
+
 
         public bool SetApplicationDirectory(string directory)
         {

source/Octopus.Tentacle/Commands/ConfigureCommand.cs

@@ -92,7 +92,6 @@ protected override void Start()
             {
                 log.Info("Removing all trusted Octopus Servers...");
                 tentacleConfiguration.Value.ResetTrustedOctopusServers();
-                VoteForRestart();
             }
 
             if (octopusToRemove.Count > 0)
@@ -101,7 +100,6 @@ protected override void Start()
                 foreach (var toRemove in octopusToRemove)
                 {
                     tentacleConfiguration.Value.RemoveTrustedOctopusServersWithThumbprint(toRemove);
-                    VoteForRestart();
                 }
             }
 
@@ -113,7 +111,6 @@ protected override void Start()
                 {
                     var config = new OctopusServerConfiguration(toAdd) { CommunicationStyle = CommunicationStyle.TentaclePassive };
                     tentacleConfiguration.Value.AddOrUpdateTrustedOctopusServer(config);
-                    VoteForRestart();
                 }
             }
 

source/Octopus.Tentacle/Communications/HalibutInitializer.cs

@@ -40,14 +40,15 @@ public void Start()
             if (configuration.NoListen)
             {
                 log.Info("Agent will not listen on any TCP ports");
-                return;
+            }
+            else
+            {
+                var endpoint = GetEndPointToListenOn();
+                halibut.Listen(endpoint);
+                log.Info("Agent listening on: " + endpoint);   
             }
 
-            var endpoint = GetEndPointToListenOn();
-
-            halibut.Listen(endpoint);
-
-            log.Info("Agent listening on: " + endpoint);
+            configuration.Changed += TentacleConfigurationChanged;
         }
 
         private void FixCommunicationStyle()
@@ -82,9 +83,7 @@ void TrustOctopusServers()
                 log.Info("The agent is not configured to trust any Octopus Servers.");
             }
         }
-
-
-
+
         void AddPollingEndpoints()
         {
             foreach (var pollingEndPoint in GetOctopusServersToPoll())
@@ -170,6 +169,21 @@ IPEndPoint GetEndPointToListenOn()
             return new IPEndPoint(address, configuration.ServicesPortNumber);
         }
 
+        void TentacleConfigurationChanged()
+        {
+            var thumbprints = GetTrustedOctopusThumbprints();
+
+            foreach (var thumbprint in thumbprints)
+            {
+                if (!halibut.IsTrusted(thumbprint))
+                {
+                    log.Info($"Agent will trust Octopus Servers with the thumbprint: {thumbprint}");
+                }
+            }
+
+            halibut.TrustOnly(thumbprints);
+        }
+
         public void Stop()
         {
         }

source/Octopus.Tentacle/Configuration/ConfigurationChangedEventHandler.cs

@@ -0,0 +1,6 @@
+using System;
+
+namespace Octopus.Tentacle.Configuration
+{
+    public delegate void ConfigurationChangedEventHandler();
+}

source/Octopus.Tentacle/Configuration/FlatDictionaryKeyValueStore.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using Newtonsoft.Json;
 using Octopus.Tentacle.Configuration.Crypto;
 using Octopus.Tentacle.Configuration.Instances;
@@ -40,7 +41,7 @@ protected FlatDictionaryKeyValueStore(JsonSerializerSettings jsonSerializerSetti
 
                 return JsonConvert.DeserializeObject<TData>((string)data, JsonSerializerSettings);
             }
-            catch (Exception e)
+            catch (Exception e) when (e is not IOException)
             {
                 if (protectionLevel == ProtectionLevel.None)
                     throw new FormatException($"Unable to parse configuration key '{name}' as a '{typeof(TData).Name}'. Value was '{valueAsString}'.", e);
@@ -74,7 +75,7 @@ protected FlatDictionaryKeyValueStore(JsonSerializerSettings jsonSerializerSetti
 
                 return (true, JsonConvert.DeserializeObject<TData>((string)data, JsonSerializerSettings));
             }
-            catch (Exception e)
+            catch (Exception e) when (e is not IOException)
             {
                 if (protectionLevel == ProtectionLevel.None)
                     throw new FormatException($"Unable to parse configuration key '{name}' as a '{typeof(TData).Name}'. Value was '{valueAsString}'.", e);

source/Octopus.Tentacle/Configuration/ITentacleConfiguration.cs

@@ -86,6 +86,8 @@ public interface ITentacleConfiguration
         bool IsRegistered { get; }
 
         void WriteTo(IWritableKeyValueStore outputStore, IEnumerable<string> excluding);
+
+        ConfigurationChangedEventHandler? Changed { get; set; }
     }
 
     public interface IWritableTentacleConfiguration : ITentacleConfiguration

source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceConfiguration.cs

@@ -2,24 +2,38 @@
 
 namespace Octopus.Tentacle.Configuration.Instances
 {
-    public class ApplicationInstanceConfiguration {
+    public class ApplicationInstanceConfiguration : IDisposable
+    {
         public ApplicationInstanceConfiguration(
             string? instanceName,
             string? configurationPath,
             IKeyValueStore? configuration,
-            IWritableKeyValueStore? writableConfiguration)
+            IWritableKeyValueStore? writableConfiguration,
+            Func<IKeyValueStore?>? loadConfigurationFunction)
         {
             InstanceName = instanceName;
             ConfigurationPath = configurationPath;
             Configuration = configuration;
             WritableConfiguration = writableConfiguration;
+
+            if (configuration != null)
+            {
+                ChangeDetectingConfiguration = new ChangeDetectingKeyValueStore(configuration, configurationPath, loadConfigurationFunction);
+            }
         }
 
         public string? ConfigurationPath { get;  }
         public string? InstanceName { get;  }
 
         public IKeyValueStore? Configuration { get; }
 
+        public ChangeDetectingKeyValueStore? ChangeDetectingConfiguration { get; }
+
         public IWritableKeyValueStore? WritableConfiguration { get; }
+
+        public void Dispose()
+        {
+            ChangeDetectingConfiguration?.Dispose();
+        }
     }
 }