Ignore more tests that rely on timing. Enable tests that usually fail…

… due to disk space issues

source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs

@@ -11,6 +11,7 @@
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
 using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
@@ -24,6 +25,7 @@ namespace Octopus.Tentacle.Tests.Integration
     /// from RPC calls when they are being retried and the rpc timeout period elapses.
     /// </summary>
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientFileTransferRetriesTimeout : IntegrationTest
     {
         readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1);

source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs

@@ -7,13 +7,15 @@
 using Octopus.Tentacle.Contracts.ClientServices;
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers;
 
 namespace Octopus.Tentacle.Tests.Integration
 {
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest
     {
         [Test]

source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs

@@ -10,6 +10,7 @@
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
 using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers;
@@ -20,6 +21,7 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes
     {
         [Test]
         [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()

source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs

@@ -218,7 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(
 
         [Test]
         [TentacleConfigurations]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             var retryDuration = TimeSpan.FromSeconds(15);

source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs

@@ -6,13 +6,15 @@
 using Octopus.Tentacle.Contracts.ClientServices;
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 
 namespace Octopus.Tentacle.Tests.Integration
 {
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest
     {
         [Test]

source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs

@@ -16,7 +16,6 @@ public class FileTransferServiceTests : IntegrationTest
     {
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             using var fileToUpload = new RandomTemporaryFileBuilder().Build();
@@ -44,7 +43,6 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC
 
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             using var fileToDownload = new RandomTemporaryFileBuilder().Build();

source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs

@@ -133,7 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati
 
         [Test]
         [TentacleConfigurations]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()

source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs

@@ -10,6 +10,13 @@ namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes
     [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
     public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest
     {
+        string Reason { get; }
+
+        public SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute(string reason)
+        {
+            Reason = reason;
+        }
+
         public void ApplyToTest(Test test)
         {
             if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored)
@@ -18,7 +25,7 @@ public void ApplyToTest(Test test)
             if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _))
             {
                 test.RunState = RunState.Skipped;
-                test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues");
+                test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}");
             }
         }
     }

source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs

@@ -28,6 +28,7 @@ namespace Octopus.Tentacle.Tests.Integration
     /// Please review any changes to the assertions made by these tests carefully.
     /// </summary>
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")]
     public class TentacleCommandLineTests : IntegrationTest
     {
         [Test]
@@ -272,7 +273,6 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu
 
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc)
         {
             var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json");

source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs

@@ -33,6 +33,7 @@ public void SetUpLocal()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void ExitCode_ShouldBeReturned()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -56,6 +57,7 @@ public void ExitCode_ShouldBeReturned()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -83,7 +85,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()
 
         [Test]
         [Retry(3)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void CancellationToken_ShouldForceKillTheProcess()
         {
             // Terminate the process after a very short time so the test doesn't run forever
@@ -116,6 +118,7 @@ public void CancellationToken_ShouldForceKillTheProcess()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void EchoHello_ShouldWriteToStdOut()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -138,6 +141,7 @@ public void EchoHello_ShouldWriteToStdOut()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void EchoError_ShouldWriteToStdErr()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -160,6 +164,7 @@ public void EchoError_ShouldWriteToStdErr()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void RunAsCurrentUser_ShouldWork()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -186,6 +191,7 @@ public void RunAsCurrentUser_ShouldWork()
         [Test]
         [WindowsTest]
         [TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments)
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))