TestPlatform.PR (#1837)

* Adding the YAML file for PR

* Removed redundant lines.

* Acceptance tests and adding changes for Ubuntu

* Removed redundant fields

* Updating the Ubuntu scripts

* Updating the shell script

* Case sensitivity for build configuration

* Removing the test for Ubuntu for now.

* Trying on hosted agents for Windows as well.

* Fixing flaky console logger tests.

* Fixing the Common platform tests

* Fix for the smoke tests run.

* Fixing the name for trx for publishing the test run

azure-pipelines.yml

@@ -0,0 +1,74 @@
+# Pipeline YAML for PR
+
+jobs:
+- job: Windows
+  pool:
+    vmImage: 'vs2017-win2016'
+  variables:
+    buildConfiguration: 'Release'
+  steps:
+  - task: BatchScript@1
+    displayName: 'Run script build.cmd'
+    inputs:
+      filename: build.cmd
+      arguments: '-verbose -configuration $(buildConfiguration)'
+      modifyEnvironment: false
+      failOnStandardError: true
+
+  - task: CmdLine@1
+    displayName: 'Delete stale pdb files'
+    inputs:
+      filename: del
+      arguments: '/S /F *.pdb'
+      workingFolder: '$(SystemRoot)'
+    continueOnError: true
+    condition: always()
+
+  - task: BatchScript@1
+    displayName: 'Run Unit Tests'
+    inputs:
+      filename: test.cmd
+      arguments: '-verbose -configuration $(buildConfiguration)'
+      modifyEnvironment: false
+      failOnStandardError: true
+
+  - task: BatchScript@1
+    displayName: 'Run Acceptance Tests'
+    inputs:
+      filename: test.cmd
+      arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451'
+      modifyEnvironment: false
+      failOnStandardError: true
+
+  - task: BatchScript@1
+    displayName: 'Run Platform Tests'
+    inputs:
+      filename: test.cmd
+      arguments: '-verbose -configuration $(buildConfiguration) -p platformtests'
+      modifyEnvironment: false
+      failOnStandardError: true
+
+  - task: BatchScript@1
+    displayName: 'Run Smoke Tests'
+    inputs:
+      filename: test.cmd
+      arguments: '-verbose -configuration $(buildConfiguration) -p smoke'
+      modifyEnvironment: false
+      failOnStandardError: true
+
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results **\*.trx'
+    inputs:
+      testResultsFormat: VSTest
+      testResultsFiles: '**\*.trx'
+    condition: succeededOrFailed()
+
+- job: Linux
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+      buildConfiguration: 'Release'
+  steps:
+  - script: ./build.sh -c $(buildConfiguration)
+    displayName: './build.sh -c $(buildConfiguration)'
+

scripts/test.ps1

@@ -237,7 +237,7 @@ function Invoke-Test
                         $testContainerPath
                     }
                 }
-                $trxLogFileName  =  [System.String]::Format("Parallel_{0}_{1}", $fx, $Script:TPT_DefaultTrxFileName)
+                $trxLogFileName  =  [System.String]::Format("Parallel_{0}_{1}_{2}", $TPT_Pattern, $fx, $Script:TPT_DefaultTrxFileName)
 
                 # Remove already existed trx file name as due to which warning will get generated and since we are expecting result in a particular format, that will break
                 $fullTrxFilePath = Join-Path $Script:TPT_TestResultsDir $trxLogFileName

scripts/test.sh

@@ -18,7 +18,6 @@ CONFIGURATION="Debug"
 TARGET_RUNTIME="ubuntu.16.04-x64"
 FAIL_FAST=false
 VERBOSE=false
-PROJECT_NAME_PATTERNS=**Unit*bin*Debug*netcoreapp1.0*UnitTests*dll
 
 while [ $# -gt 0 ]; do
     lowerI="$(echo ${1:-} | awk '{print tolower($0)}')"
@@ -49,6 +48,7 @@ done
 #
 # Variables
 #
+PROJECT_NAME_PATTERNS=**Unit*bin*$CONFIGURATION*netcoreapp1.0*UnitTests*dll
 TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P)
 TP_TOOLS_DIR="$TP_ROOT_DIR/tools"
 TP_PACKAGES_DIR="$TP_ROOT_DIR/packages"

test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj

@@ -6,6 +6,7 @@
   </PropertyGroup>
   <Import Project="$(TestPlatformRoot)scripts/build/TestPlatform.Settings.targets" />
   <PropertyGroup>
+    <OutputType Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">Exe</OutputType>
     <TargetFrameworks>netcoreapp1.0;net451</TargetFrameworks>
     <AssemblyName>Microsoft.TestPlatform.Common.PlatformTests</AssemblyName>
     <EnableCodeAnalysis>true</EnableCodeAnalysis>

test/Microsoft.TestPlatform.Common.PlatformTests/Program.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.UnitTests
+{
+    public static class Program
+    {
+        public static void Main(string[] args)
+        {
+        }
+    }
+}

test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj

@@ -6,9 +6,9 @@
   </PropertyGroup>
   <Import Project="$(TestPlatformRoot)scripts/build/TestPlatform.Settings.targets" />
   <PropertyGroup>
-    <OutputType Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">Exe</OutputType>
+    <OutputType Condition=" '$(TargetFramework)' != 'net451' ">Exe</OutputType>
     <AssemblyName>Microsoft.TestPlatform.SmokeTests</AssemblyName>
-    <TargetFrameworks>netcoreapp1.0;net451</TargetFrameworks>
+    <TargetFrameworks>netcoreapp1.0;netcoreapp2.0;net451</TargetFrameworks>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.TestPlatform.TestUtilities\Microsoft.TestPlatform.TestUtilities.csproj" />

test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs

@@ -144,7 +144,8 @@ public void TestMessageHandlerShouldWriteToConsoleWhenTestRunMessageIsRaised()
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Informational, "Informational123"));
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, "Error123"));
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, "Warning123"));
-
+            loggerEvents.WaitForEventCompletion();
+
             // Added this for synchronization
             SpinWait.SpinUntil(() => count == 3, 300);
 
@@ -163,6 +164,7 @@ public void TestMessageHandlerShouldWriteToConsoleWhenTestDiscoveryMessageIsRais
             loggerEvents.RaiseDiscoveryMessage(new TestRunMessageEventArgs(TestMessageLevel.Informational, "Informational123"));
             loggerEvents.RaiseDiscoveryMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, "Error123"));
             loggerEvents.RaiseDiscoveryMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, "Warning123"));
+            loggerEvents.WaitForEventCompletion();
 
             // Added this for synchronization
             SpinWait.SpinUntil(() => count == 3, 300);
@@ -223,6 +225,7 @@ public void TestResultHandlerShouldShowStdOutMessagesBannerIfStdOutIsNotEmpty()
             testresult.Messages.Add(testResultMessage);
 
             loggerEvents.RaiseTestResult(new TestResultEventArgs(testresult));
+            loggerEvents.WaitForEventCompletion();
 
             // Added this for synchronization
             SpinWait.SpinUntil(() => count == 2, 300);
@@ -253,6 +256,7 @@ public void NormalVerbosityShowNotStdOutMessagesForPassedTests()
 
             // Raise an event on mock object
             loggerEvents.RaiseTestResult(new TestResultEventArgs(testresult));
+            loggerEvents.WaitForEventCompletion();
 
             // Verify
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.StdOutMessagesBanner, OutputLevel.Information), Times.Never());
@@ -308,7 +312,8 @@ public void TestRunErrorMessageShowShouldTestRunFailed()
             // Act. Raise an event on mock object
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, message));
             loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock<ITestRunStatistics>().Object, false, false, null, new Collection<AttachmentSet>(), TimeSpan.FromSeconds(1)));
-
+            loggerEvents.WaitForEventCompletion();
+
             // Verify
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunFailed, OutputLevel.Error), Times.Once());
             this.mockOutput.Verify(o => o.WriteLine(message, OutputLevel.Error), Times.Once());
@@ -331,6 +336,7 @@ public void InQuietModeTestErrorMessageShowShouldShowTestRunFailed()
             // Act. Raise an event on mock object
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, message));
             loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock<ITestRunStatistics>().Object, false, false, null, new Collection<AttachmentSet>(), TimeSpan.FromSeconds(1)));
+            loggerEvents.WaitForEventCompletion();
 
             // Verify
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunFailed, OutputLevel.Error), Times.Once());
@@ -354,6 +360,7 @@ public void InQuietModeTestWarningMessageShouldNotShow()
             // Act. Raise an event on mock object
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, message));
             loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock<ITestRunStatistics>().Object, false, false, null, new Collection<AttachmentSet>(), TimeSpan.FromSeconds(1)));
+            loggerEvents.WaitForEventCompletion();
 
             // Verify
             this.mockOutput.Verify(o => o.WriteLine(message, OutputLevel.Warning), Times.Never());
@@ -378,6 +385,7 @@ public void InNormalModeTestWarningAndErrorMessagesShouldShow()
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Warning, message));
             loggerEvents.RaiseTestRunMessage(new TestRunMessageEventArgs(TestMessageLevel.Error, errorMessage));
             loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock<ITestRunStatistics>().Object, false, false, null, new Collection<AttachmentSet>(), TimeSpan.FromSeconds(1)));
+            loggerEvents.WaitForEventCompletion();
 
             // Verify
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunFailed, OutputLevel.Error), Times.Once());
@@ -405,6 +413,7 @@ public void TestResultHandlerShouldNotShowStdOutMessagesBannerIfStdOutIsEmpty()
             testresult.Messages.Add(testResultMessage);
 
             loggerEvents.RaiseTestResult(new TestResultEventArgs(testresult));
+            loggerEvents.WaitForEventCompletion();
 
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.StdOutMessagesBanner, OutputLevel.Information), Times.Never());
         }
@@ -454,6 +463,7 @@ public void TestResultHandlerShouldNotShowStdErrMessagesBannerIfStdErrIsEmpty()
             testresult.Messages.Add(testResultMessage);
 
             loggerEvents.RaiseTestResult(new TestResultEventArgs(testresult));
+            loggerEvents.WaitForEventCompletion();
 
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.StdErrMessagesBanner, OutputLevel.Information), Times.Never());
         }
@@ -505,6 +515,7 @@ public void TestResultHandlerShouldNotShowAdditionalInfoBannerIfAdditionalInfoIs
             testresult.Messages.Add(testResultMessage);
 
             loggerEvents.RaiseTestResult(new TestResultEventArgs(testresult));
+            loggerEvents.WaitForEventCompletion();
 
             this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.AddnlInfoMessagesBanner, OutputLevel.Information), Times.Never());
         }
@@ -614,6 +625,7 @@ public void TestResultHandlerShouldWriteToNoTestResultForQuietVerbosity()
             {
                 loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult));
             }
+            loggerEvents.WaitForEventCompletion();
 
             this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName"), OutputLevel.Information), Times.Never);
             this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.FailedTestIndicator, "TestName"), OutputLevel.Information), Times.Never);