Restructure tests and get them working under .NET 6.0

TestCentric.Extensibility.sln

@@ -37,6 +37,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
 		.github\workflows\testcentric-extensibility.ci.yml = .github\workflows\testcentric-extensibility.ci.yml
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestCentric.Extensibility.FakeExtensions", "src\testcentric.extensibility.fakes\TestCentric.Extensibility.FakeExtensions.csproj", "{B15F3AB7-7930-4FE5-A2CC-DB61F8CFC275}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -55,6 +57,10 @@ Global
 		{BE988F0A-3337-4DC6-8915-138C4875FD04}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BE988F0A-3337-4DC6-8915-138C4875FD04}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BE988F0A-3337-4DC6-8915-138C4875FD04}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B15F3AB7-7930-4FE5-A2CC-DB61F8CFC275}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B15F3AB7-7930-4FE5-A2CC-DB61F8CFC275}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B15F3AB7-7930-4FE5-A2CC-DB61F8CFC275}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B15F3AB7-7930-4FE5-A2CC-DB61F8CFC275}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/testcentric.extensibility.fakes/TestCentric.Extensibility.FakeExtensions.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+	<PropertyGroup>
+		<TargetFrameworks>net20;net462;netstandard2.0</TargetFrameworks>
+		<RootNamespace>TestCentric.Extensibility</RootNamespace>
+		<SignAssembly>true</SignAssembly>
+		<AssemblyOriginatorKeyFile>testcentric.snk</AssemblyOriginatorKeyFile>
+		<OutputPath>..\..\bin\$(Configuration)\fakes</OutputPath>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\TestCentric.Extensibility.Api\TestCentric.Extensibility.Api.csproj" />
+		<PackageReference Include="TestCentric.Engine.Api" Version="2.0.0-beta5" />
+		<PackageReference Include="NUnit.Engine.Api" Version="3.16.2" />
+	</ItemGroup>
+
+</Project>

src/testcentric.extensibility.tests/ExtensionManagerTestBase.cs

@@ -3,8 +3,6 @@
 // Licensed under the MIT License. See LICENSE file in root directory.
 // ***********************************************************************
 
-// TODO: Get this test working under .NET 6.0
-#if NETFRAMEWORK
 using NSubstitute;
 using NUnit.Framework;
 using System;
@@ -26,6 +24,7 @@ public abstract class ExtensionManagerTestBase
     {
         protected static readonly Assembly THIS_ASSEMBLY = typeof(ExtensionManager_NewApi).Assembly;
         protected static readonly string THIS_ASSEMBLY_DIRECTORY = Path.GetDirectoryName(THIS_ASSEMBLY.Location);
+
         protected static readonly Assembly TESTCENTRIC_ENGINE_API = typeof(TestCentric.Engine.Extensibility.IDriverFactory).Assembly;
         protected static readonly Assembly NUNIT_ENGINE_API = typeof(NUnit.Engine.Extensibility.IDriverFactory).Assembly;
 
@@ -158,143 +157,40 @@ public void DisabledExtensionMayBeEnabled()
                    .And.Property(nameof(ExtensionNode.Enabled)).True);
         }
 
-        [Test]
-        public void SkipsGracefullyLoadingOtherFrameworkExtensionAssembly()
-        {
-            //May be null on mono
-            Assume.That(Assembly.GetEntryAssembly(), Is.Not.Null, "Entry assembly is null, framework loading validation will be skipped.");
-
 #if NETCOREAPP
-        var assemblyName = Path.Combine(GetSiblingDirectory("net462"), "TestCentric.Engine.Api.dll");
+        [TestCase("netstandard2.0", ExpectedResult = true)]
+        [TestCase("net462", ExpectedResult = false)]
+        [TestCase("net20", ExpectedResult = false)]
+#elif NET40_OR_GREATER
+        [TestCase("netstandard2.0", ExpectedResult = false)]
+        [TestCase("net462", ExpectedResult = true)]
+        [TestCase("net20", ExpectedResult = true)]
 #else
-        var assemblyName = Path.Combine(GetSiblingDirectory("net6.0"), "TestCentric.Engine.Api.dll");
+        [TestCase("netstandard2.0", ExpectedResult = false)]
+        [TestCase("net462", ExpectedResult = false)]
+        [TestCase("net20", ExpectedResult = true)]
 #endif
-            Assert.That(assemblyName, Does.Exist);
-
-            var manager = new ExtensionManager();
-            //manager.FindExtensionPoints(typeof(DriverService).Assembly);
-            manager.FindExtensionPoints(typeof(Engine.ITestEngine).Assembly);
-            var extensionAssembly = new ExtensionAssembly(assemblyName, false);
-
-            Assert.That(() => manager.FindExtensionsInAssembly(extensionAssembly), Throws.Nothing);
-        }
-
-        [TestCaseSource(nameof(ValidCombos))]
-        public void ValidTargetFrameworkCombinations(FrameworkCombo combo)
-        {
-            Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
-                Is.True);
-        }
-
-        [TestCaseSource(nameof(InvalidTargetFrameworkCombos))]
-        public void InvalidTargetFrameworkCombinations(FrameworkCombo combo)
-        {
-            Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
-                Is.False);
-        }
-
-        [TestCaseSource(nameof(InvalidRunnerCombos))]
-        public void InvalidRunnerTargetFrameworkCombinations(FrameworkCombo combo)
+        public bool LoadTargetFramework(string tfm)
         {
-            Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
-                Throws.Exception.And.Message.Contains("not .NET Standard"));
+            return ExtensionManager.CanLoadTargetFramework(THIS_ASSEMBLY, FakeExtensions(tfm));
         }
 
         #endregion
 
-        // ExtensionAssembly is internal, so cannot be part of the public test parameters
-        public struct FrameworkCombo
-        {
-            internal Assembly RunnerAssembly { get; }
-            internal ExtensionAssembly ExtensionAssembly { get; }
-
-            internal FrameworkCombo(Assembly runnerAsm, ExtensionAssembly extensionAsm)
-            {
-                RunnerAssembly = runnerAsm;
-                ExtensionAssembly = extensionAsm;
-            }
-
-            public override string ToString() =>
-                $"{RunnerAssembly.GetName()}:{ExtensionAssembly.AssemblyName}";
-        }
-
-        public static IEnumerable<TestCaseData> ValidCombos()
-        {
-#if NETCOREAPP
-            Assembly netstandard = typeof(ExtensionManager).Assembly;
-            Assembly netcore = Assembly.GetExecutingAssembly();
-
-            var extNetStandard = new ExtensionAssembly(netstandard.Location, false);
-            var extNetCore = new ExtensionAssembly(netcore.Location, false);
-
-            yield return new TestCaseData(new FrameworkCombo(netcore, extNetStandard)).SetName("ValidCombo(.NET Core, .NET Standard)");
-            yield return new TestCaseData(new FrameworkCombo(netcore, extNetCore)).SetName("ValidCombo(.NET Core, .Net Core)");
-#else
-            Assembly netFramework = typeof(ExtensionManager).Assembly;
-
-            var extNetFramework = new ExtensionAssembly(netFramework.Location, false);
-            var extNetStandard = new ExtensionAssembly(Path.Combine(TestContext.CurrentContext.TestDirectory, "TestCentric.Engine.Api.dll"), false);
-
-            yield return new TestCaseData(new FrameworkCombo(netFramework, extNetFramework)).SetName("ValidCombo(.NET Framework, .NET Framework)");
-            //yield return new TestCaseData(new FrameworkCombo(netFramework, extNetStandard)).SetName("ValidCombo(.NET Framework, .NET Standard)");
-#endif
-        }
-
-        public static IEnumerable<TestCaseData> InvalidTargetFrameworkCombos()
-        {
-#if NETCOREAPP
-            Assembly netstandard = typeof(ExtensionManager).Assembly;
-            Assembly netcore = Assembly.GetExecutingAssembly();
-
-            var extNetStandard = new ExtensionAssembly(netstandard.Location, false);
-            var extNetCore = new ExtensionAssembly(netcore.Location, false);
-            var extNetFramework = new ExtensionAssembly(Path.Combine(GetNetFrameworkSiblingDirectory(), "testcentric.engine.core.dll"), false);
-
-            yield return new TestCaseData(new FrameworkCombo(netcore, extNetFramework)).SetName("InvalidCombo(.NET Core, .NET Framework)");
-#else
-            Assembly netFramework = typeof(ExtensionManager).Assembly;
-
-
-            var netCoreAppDir = GetSiblingDirectory("net6.0");
-            var extNetStandard = new ExtensionAssembly(Path.Combine(netCoreAppDir, "TestCentric.Engine.Api.dll"), false);
-            var extNetCoreApp = new ExtensionAssembly(Path.Combine(netCoreAppDir, "TestCentric.Extensibility.dll"), false);
-
-            yield return new TestCaseData(new FrameworkCombo(netFramework, extNetCoreApp)).SetName("InvalidCombo(.NET Framework, .NET Core)");
-#endif
-
-        }
-
-        public static IEnumerable<TestCaseData> InvalidRunnerCombos()
-        {
-#if NETCOREAPP
-            Assembly netstandard = typeof(ExtensionManager).Assembly;
-            Assembly netcore = Assembly.GetExecutingAssembly();
-
-            var extNetStandard = new ExtensionAssembly(netstandard.Location, false);
-            var extNetCore = new ExtensionAssembly(netcore.Location, false);
-            var extNetFramework = new ExtensionAssembly(Path.Combine(GetNetFrameworkSiblingDirectory(), "testcentric.extensibility.tests.dll"), false);
-
-            yield return new TestCaseData(new FrameworkCombo(netstandard, extNetStandard)).SetName("InvalidCombo(.NET Standard, .NET Standard)");
-            yield return new TestCaseData(new FrameworkCombo(netstandard, extNetCore)).SetName("InvalidCombo(.NET Standard, .NET Core)");
-            yield return new TestCaseData(new FrameworkCombo(netstandard, extNetFramework)).SetName("InvalidCombo(.NET Standard, .NET Framework)");
-#else
-            return new List<TestCaseData>();
-#endif
-        }
+        private const string FAKE_EXTENSIONS_FILENAME = "TestCentric.Extensibility.FakeExtensions.dll";
+        private static readonly string FAKE_EXTENSIONS_PARENT_DIRECTORY =
+            Path.Combine(new DirectoryInfo(THIS_ASSEMBLY_DIRECTORY).Parent.Parent.FullName, "fakes");
 
         /// <summary>
-        /// Returns a directory in the parent directory that the current test assembly is in. This
-        /// is used to load assemblies that target different frameworks than the current tests. So
-        /// if these tests are in bin\release\net35 and dir is netstandard2.0, this will return
-        /// bin\release\netstandard2.0.
+        /// Returns an ExtensionAssembly referring to a particular build of the fake test extensions
+        /// assembly based on the argument provided.
         /// </summary>
-        /// <param name="dir">The sibling directory</param>
+        /// <param name="tfm">A test framework moniker. Must be one for which the fake extensions are built.</param>
         /// <returns></returns>
-        private static string GetSiblingDirectory(string dir)
+        private static ExtensionAssembly FakeExtensions(string tfm)
         {
-            var file = new FileInfo(typeof(ExtensionManagerTestBase).Assembly.Location);
-            return Path.Combine(file.Directory.Parent.FullName, dir);
+            return new ExtensionAssembly(
+                Path.Combine(FAKE_EXTENSIONS_PARENT_DIRECTORY, Path.Combine(tfm, FAKE_EXTENSIONS_FILENAME)), false);
         }
     }  
 }
-#endif

src/testcentric.extensibility.tests/ExtensionManager_NewApi.cs

@@ -3,7 +3,6 @@
 // Licensed under the MIT License. See LICENSE file in root directory.
 // ***********************************************************************
 
-#if NETFRAMEWORK
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
@@ -40,4 +39,3 @@ public void CreateManager()
         }
     }
 }
-#endif

src/testcentric.extensibility.tests/ExtensionManger_OldApi.cs

@@ -3,8 +3,6 @@
 // Licensed under the MIT License. See LICENSE file in root directory.
 // ***********************************************************************
 
-// TODO: Get this test working under .NET 8.0
-#if NETFRAMEWORK
 using System;
 using System.Linq;
 using System.IO;
@@ -31,7 +29,10 @@ public ExtensionManager_OldApi(string defaultTypeExtensionsPath)
         public void CreateManager()
         {
 #pragma warning disable 612, 618
-            ExtensionManager = new ExtensionManager(TESTCENTRIC_ENGINE_API, NUNIT_ENGINE_API) { InitialAddinsDirectory = THIS_ASSEMBLY_DIRECTORY };
+            ExtensionManager = new ExtensionManager(TESTCENTRIC_ENGINE_API, NUNIT_ENGINE_API) 
+            { 
+                InitialAddinsDirectory = THIS_ASSEMBLY_DIRECTORY 
+            };
 
             if (PrefixWasProvided)
                 ExtensionManager.DefaultTypeExtensionPrefix = DefaultTypeExtensionsPath;
@@ -43,4 +44,3 @@ public void CreateManager()
         }
     }  
 }
-#endif

src/testcentric.extensibility.tests/Properties/launchSettings.json

@@ -1,8 +1,7 @@
 {
   "profiles": {
     "TestCentric.Extensibility.Tests": {
-      "commandName": "Project",
-      "commandLineArgs": "--test TestCentric.Extensibility.ExtensionManagerTests"
+      "commandName": "Project"
     }
   }
 }

src/testcentric.extensibility.tests/testcentric.extensibility.tests.addins

@@ -1,3 +1,2 @@
 # Used for unit testing - not part of distributed package
-testcentric.extensibility.tests.dll
-testcentric.extensibility.tests.exe
+../../fakes/*/TestCentric.Extensibility.FakeExtensions.dll

src/testcentric.extensibility/ExtensionManager.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Reflection;
 using TestCentric.Metadata;
@@ -386,6 +387,14 @@ private void ProcessCandidateAssembly(string filePath, bool fromWildCard)
                 {
                     var candidate = new ExtensionAssembly(filePath, fromWildCard);
 
+                    // Make sure we can load this assembly
+                    if (!CanLoadTargetFramework(Assembly.GetEntryAssembly(), candidate))
+                    {
+                        log.Info($"{candidate.FilePath} cannot be loaded on this runtime");
+                        return;
+                    }
+
+                    // Check to see if this is a duplicate
                     for (int i = 0; i < _extensionAssemblies.Count; i++)
                     {
                         var assembly = _extensionAssemblies[i];
@@ -399,7 +408,7 @@ private void ProcessCandidateAssembly(string filePath, bool fromWildCard)
                             }
                             else
                                 log.Info($"  Assembly: {Path.GetFileName(filePath)} ,fromWildCard = {fromWildCard}, duplicate ignored");
-                            
+
                             return;
                         }
                     }
@@ -583,10 +592,10 @@ internal bool CanLoadTargetFramework(Assembly runnerAsm, ExtensionAssembly exten
                 case ".NETCoreApp":
                     switch (extensionFrameworkName.Identifier)
                     {
-                        case ".NetStandard":
-                        case ".NetCoreApp":
+                        case ".NETStandard":
+                        case ".NETCoreApp":
                             return true;
-                        case ".NetFramework":
+                        case ".NETFramework":
                         default:
                             log.Info($".NET Core runners require .NET Core or .NET Standard extension for {extensionAsm.FilePath}");
                             return false;
@@ -600,7 +609,7 @@ internal bool CanLoadTargetFramework(Assembly runnerAsm, ExtensionAssembly exten
                         // For .NET Framework calling .NET Standard, we only support if framework is 4.7.2 or higher
                         case ".NETStandard":
                             return extensionFrameworkName.Version >= new Version(4, 7, 2);
-                        case ".NetCoreApp":
+                        case ".NETCoreApp":
                         default:
                             log.Info($".NET Framework runners cannot load .NET Core extension {extensionAsm.FilePath}");
                             return false;