Device test runner

azure-pipelines.yml

@@ -6,8 +6,9 @@ variables:
   NugetPackageVersionCamera: '$(CurrentSemanticVersion)'
   NugetPackageVersionMediaElement: '$(CurrentSemanticVersion)'
   NugetPackageVersionMaps: '$(CurrentSemanticVersion)'
-  TOOLKIT_NET_VERSION: '8.0.303'
+  TOOLKIT_NET_VERSION: '8.0.402'
   LATEST_NET_VERSION: '8.0.x'
+  TFM_NET_VERSION: 'net8.0'
   PathToLibrarySolution: 'src/CommunityToolkit.Maui.sln'
   PathToSamplesSolution: 'samples/CommunityToolkit.Maui.Sample.sln'
   PathToCommunityToolkitCsproj: 'src/CommunityToolkit.Maui/CommunityToolkit.Maui.csproj'
@@ -17,6 +18,8 @@ variables:
   PathToCommunityToolkitMapsCsproj: 'src/CommunityToolkit.Maui.Maps/CommunityToolkit.Maui.Maps.csproj'
   PathToCommunityToolkitSampleCsproj: 'samples/CommunityToolkit.Maui.Sample/CommunityToolkit.Maui.Sample.csproj'
   PathToCommunityToolkitUnitTestCsproj: 'src/CommunityToolkit.Maui.UnitTests/CommunityToolkit.Maui.UnitTests.csproj'
+  PathToCommunityToolkitDeviceTestCsproj: 'src/CommunityToolkit.Maui.DeviceTests/CommunityToolkit.Maui.DeviceTests.csproj'
+  PathToCommunityToolkitDeviceTestBin: 'src/CommunityToolkit.Maui.DeviceTests/bin'
   PathToCommunityToolkitAnalyzersCsproj: 'src/CommunityToolkit.Maui.Analyzers/CommunityToolkit.Maui.Analyzers.csproj'
   PathToCommunityToolkitCameraAnalyzersCsproj: 'src/CommunityToolkit.Maui.Camera.Analyzers/CommunityToolkit.Maui.Camera.Analyzers.csproj'
   PathToCommunityToolkitMediaElementAnalyzersCsproj: 'src/CommunityToolkit.Maui.MediaElement.Analyzers/CommunityToolkit.Maui.MediaElement.Analyzers.csproj'
@@ -29,7 +32,7 @@ variables:
   PathToCommunityToolkitAnalyzersBenchmarkCsproj: 'src/CommunityToolkit.Maui.Analyzers.Benchmarks/CommunityToolkit.Maui.Analyzers.Benchmarks.csproj'
   DotNetMauiRollbackFile: 'https://maui.blob.core.windows.net/metadata/rollbacks/8.0.6.json'
   CommunityToolkitSampleApp_Xcode_Version: '16'
-  CommunityToolkitLibrary_Xcode_Version: '15.4'
+  CommunityToolkitLibrary_Xcode_Version: '16'
 
 trigger:
   branches:
@@ -218,6 +221,11 @@ jobs:
           .\workload-install.ps1
         displayName: Install Tizen Workload
 
+      - task: CmdLine@2
+        displayName: 'Install Microsoft.DotNet.XHarness.CLI'
+        inputs:
+          script:  dotnet tool install Microsoft.DotNet.XHarness.CLI --global --version "*-*" --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json
+
       # Print Information on the .NET SDK Used By the CI Build Host
       # These logs are useful information when debugging CI Builds
       # Note: This step doesn't execute nor modify any code; it is strictly used for logging + debugging purposes
@@ -266,13 +274,6 @@ jobs:
         inputs:
           script: 'dotnet build -c Release $(PathToCommunityToolkitSourceGeneratorsInternalCsproj)'
 
-      - task: CmdLine@2
-        displayName: 'Run CommunityToolkit.Maui.Analyzers.UnitTests'
-        env:
-          VSTEST_TESTHOST_SHUTDOWN_TIMEOUT: 1100 # Fixes "The active test run was aborted. Reason: Test host process crashed" https://dev.azure.com/dotnet/CommunityToolkit/_build/results?buildId=109660&view=logs&j=3f96dcf5-6e1e-5485-3200-c557d5216be3&t=12286b69-c788-55db-0a8c-ef899858fbe6&l=76 (source: https://github.com/microsoft/vstest/issues/2952#issuecomment-2234253765): $(sauceUsername)
-        inputs:
-          script: 'dotnet test $(PathToCommunityToolkitAnalyzersUnitTestCsproj) -c Release'
-
       - task: CmdLine@2
         displayName: 'Build CommunityToolkit.Maui.Core'
         inputs:
@@ -298,13 +299,55 @@ jobs:
         inputs:
           script: 'dotnet build $(PathToCommunityToolkitMapsCsproj) -c Release -p:PackageVersion=$(NugetPackageVersionMaps)'
 
+      - task: CmdLine@2
+        displayName: 'Run CommunityToolkit.Maui.Analyzers.UnitTests'
+        env:
+          VSTEST_TESTHOST_SHUTDOWN_TIMEOUT: 1100 # Fixes "The active test run was aborted. Reason: Test host process crashed" https://dev.azure.com/dotnet/CommunityToolkit/_build/results?buildId=109660&view=logs&j=3f96dcf5-6e1e-5485-3200-c557d5216be3&t=12286b69-c788-55db-0a8c-ef899858fbe6&l=76 (source: https://github.com/microsoft/vstest/issues/2952#issuecomment-2234253765): $(sauceUsername)
+        inputs:
+          script: 'dotnet test $(PathToCommunityToolkitAnalyzersUnitTestCsproj) -c Release'
+
       - task: CmdLine@2
         displayName: 'Run CommunityToolkit.Maui.UnitTests'
         env:
           VSTEST_TESTHOST_SHUTDOWN_TIMEOUT: 1100 # Fixes "The active test run was aborted. Reason: Test host process crashed" https://dev.azure.com/dotnet/CommunityToolkit/_build/results?buildId=109660&view=logs&j=3f96dcf5-6e1e-5485-3200-c557d5216be3&t=12286b69-c788-55db-0a8c-ef899858fbe6&l=76 (source: https://github.com/microsoft/vstest/issues/2952#issuecomment-2234253765): $(sauceUsername)
         inputs:
           script: 'dotnet test $(PathToCommunityToolkitUnitTestCsproj) -c Release --settings ".runsettings" --collect "XPlat code coverage" --logger trx --results-directory $(Agent.TempDirectory)'
 
+      # Currently failing "Failed to find/create suitable simulator"
+      - task: CmdLine@2
+        condition: eq(variables['Agent.OS'], 'Darwin') # Only run this step on macOS
+        displayName: 'Run iOS UI Tests'
+        inputs:
+          script: |
+            dotnet build $(PathToCommunityToolkitDeviceTestCsproj) -r iossimulator-x64 -f $(TFM_NET_VERSION)-ios -c Release
+            xharness apple test --target ios-simulator-64 --app $(PathToCommunityToolkitDeviceTestBin)/Release/$(TFM_NET_VERSION)-ios/iossimulator-x64/CommunityToolkit.Maui.DeviceTests.app --output-directory $(Agent.TempDirectory)
+
+
+      - task: CmdLine@2
+        condition: eq(variables['Agent.OS'], 'Darwin') # Only run this step on macOS
+        displayName: 'Run MacCatalyst UI Tests'
+        inputs:
+          script: |            
+            dotnet build $(PathToCommunityToolkitDeviceTestCsproj) -r maccatalyst-x64 -f $(TFM_NET_VERSION)-maccatalyst -c Release
+            xharness apple test --target maccatalyst --app $(PathToCommunityToolkitDeviceTestBin)/Release/$(TFM_NET_VERSION)-maccatalyst/maccatalyst-x64/CommunityToolkit.Maui.DeviceTests.app --output-directory $(Agent.TempDirectory)
+
+     # Currently failing: Failed to find compatible device: x86_64
+      - task: CmdLine@2
+        displayName: 'Run Android UI Tests'
+        inputs:
+          script: |
+            dotnet publish $(PathToCommunityToolkitDeviceTestCsproj) -r android-x64 -f $(TFM_NET_VERSION)-android -c Release
+
+            dotnet tool install --global AndroidSDK.Tool
+            android device list --format json
+            android sdk list
+            android sdk install --package emulator --package "system-images;android-33;google_apis;x86_64"
+            android avd create --name TestRunnerEmulator --sdk "system-images;android-33;google_apis;x86_64" --device pixel
+            android avd start --name TestRunnerEmulator --wait-boot
+
+            xharness android test --app $(PathToCommunityToolkitDeviceTestBin)/Release/$(TFM_NET_VERSION)-android/android-x64/publish/com.companyname.communitytoolkit.maui.devicetests-Signed.apk --package-name com.companyname.communitytoolkit.maui.devicetests --instrumentation devicerunners.xharness.maui.XHarnessInstrumentation --output-directory $(Agent.TempDirectory)'
+            xharness android adb -- emu kill
+
       - task: PublishTestResults@2
         condition: eq(variables['Agent.OS'], 'Windows_NT') # Only run this step on Windows
         displayName: 'Publish Test Results'

samples/CommunityToolkit.Maui.Sample.sln

@@ -53,6 +53,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "..\src\CommunityToolkit.Maui.Analyzers.Benchmarks\CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CommunityToolkit.Maui.DeviceTests", "..\src\CommunityToolkit.Maui.DeviceTests\CommunityToolkit.Maui.DeviceTests.csproj", "{9CA70DAD-C6AE-4986-9F85-47FBE444071D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -131,6 +133,10 @@ Global
 		{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9CA70DAD-C6AE-4986-9F85-47FBE444071D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9CA70DAD-C6AE-4986-9F85-47FBE444071D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9CA70DAD-C6AE-4986-9F85-47FBE444071D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9CA70DAD-C6AE-4986-9F85-47FBE444071D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -148,6 +154,7 @@ Global
 		{372D6A40-A4E0-434A-A463-C001441C68EB} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
 		{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
 		{B80F59B7-276C-4A55-A8DD-54587C8BC3D2} = {ED5A9C0B-D270-442D-BABE-F4FF622926C8}
+		{9CA70DAD-C6AE-4986-9F85-47FBE444071D} = {9F7D54C0-EA17-409A-804F-B2E8D7F4A7F3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {1E9E61C1-5CB7-4C8E-87BA-6C1D38238679}

src/CommunityToolkit.Maui.DeviceTests/App.xaml

@@ -0,0 +1,14 @@
+<?xml version = "1.0" encoding = "UTF-8" ?>
+<Application xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+             xmlns:local="clr-namespace:CommunityToolkit.Maui.DeviceTests"
+             x:Class="CommunityToolkit.Maui.DeviceTests.App">
+    <Application.Resources>
+        <ResourceDictionary>
+            <ResourceDictionary.MergedDictionaries>
+                <ResourceDictionary Source="Resources/Styles/Colors.xaml" />
+                <ResourceDictionary Source="Resources/Styles/Styles.xaml" />
+            </ResourceDictionary.MergedDictionaries>
+        </ResourceDictionary>
+    </Application.Resources>
+</Application>

src/CommunityToolkit.Maui.DeviceTests/App.xaml.cs

@@ -0,0 +1,11 @@
+namespace CommunityToolkit.Maui.DeviceTests;
+
+public partial class App : Application
+{
+	public App()
+	{
+		InitializeComponent();
+
+		MainPage = new AppShell();
+	}
+}

src/CommunityToolkit.Maui.DeviceTests/AppShell.xaml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Shell
+    x:Class="CommunityToolkit.Maui.DeviceTests.AppShell"
+    xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+    xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+    xmlns:local="clr-namespace:CommunityToolkit.Maui.DeviceTests"
+    Shell.FlyoutBehavior="Disabled">
+
+    <ShellContent
+        Title="Home"
+        ContentTemplate="{DataTemplate local:MainPage}"
+        Route="MainPage" />
+
+</Shell>

src/CommunityToolkit.Maui.DeviceTests/AppShell.xaml.cs

@@ -0,0 +1,9 @@
+namespace CommunityToolkit.Maui.DeviceTests;
+
+public partial class AppShell : Shell
+{
+	public AppShell()
+	{
+		InitializeComponent();
+	}
+}

src/CommunityToolkit.Maui.DeviceTests/BaseUITest.cs

@@ -0,0 +1,49 @@
+using Xunit;
+
+namespace CommunityToolkit.Maui.DeviceTests;
+[Collection("UITests")]
+public abstract class UITests<T> : IAsyncLifetime
+	where T : Page
+{
+	protected T CurrentPage { get; private set; } = default!;
+
+	protected IMauiContext MauiContext { get; private set; } = default!;
+
+	public async Task InitializeAsync()
+	{
+		Routing.RegisterRoute("uitests", typeof(T));
+
+		await Shell.Current.GoToAsync("uitests");
+
+		CurrentPage = (T)Shell.Current.CurrentPage;
+		MauiContext = CurrentPage.Handler!.MauiContext!;
+		if (CurrentPage.IsLoaded)
+		{
+			return;
+		}
+
+		var tcs = new TaskCompletionSource();
+		CurrentPage.Loaded += OnLoaded;
+
+		await Task.WhenAny(tcs.Task, Task.Delay(1000));
+
+		CurrentPage.Loaded -= OnLoaded;
+
+		Assert.True(CurrentPage.IsLoaded);
+
+		void OnLoaded(object? sender, EventArgs e)
+		{
+			CurrentPage.Loaded -= OnLoaded;
+			tcs.SetResult();
+		}
+	}
+
+	public async Task DisposeAsync()
+	{
+		CurrentPage = null!;
+
+		await Shell.Current.GoToAsync("..");
+
+		Routing.UnRegisterRoute("uitests");
+	}
+}

src/CommunityToolkit.Maui.DeviceTests/CommunityToolkit.Maui.DeviceTests.csproj

@@ -0,0 +1,70 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>$(NetVersion)-ios;$(NetVersion)-android;$(NetVersion)-maccatalyst</TargetFrameworks>
+    <TargetFrameworks Condition="$([MSBuild]::IsOSPlatform('windows'))">$(TargetFrameworks);$(NetVersion)-windows10.0.19041.0</TargetFrameworks>
+    <!-- Uncomment to also build the tizen app. You will need to install tizen by following this: https://github.com/Samsung/Tizen.NET -->
+    <!-- <TargetFrameworks>$(TargetFrameworks);$(NetVersion)-tizen</TargetFrameworks> -->
+    <OutputType>Exe</OutputType>
+    <RootNamespace>CommunityToolkit.Maui.DeviceTests</RootNamespace>
+    <UseMaui>true</UseMaui>
+    <SingleProject>true</SingleProject>
+
+    <!-- Display name -->
+    <ApplicationTitle>CommunityToolkit.Maui.DeviceTests</ApplicationTitle>
+
+    <!-- App Identifier -->
+    <ApplicationId>com.microsoft.CommunityToolkit.Maui.DeviceTests</ApplicationId>
+    <ApplicationIdGuid>b9372eea-e8b8-45fb-b785-6ffb8eb6c099</ApplicationIdGuid>
+
+    <!-- Versions -->
+    <ApplicationDisplayVersion>1.0</ApplicationDisplayVersion>
+    <ApplicationVersion>1</ApplicationVersion>
+
+    <SupportedOSPlatformVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'ios'">11.0</SupportedOSPlatformVersion>
+    <SupportedOSPlatformVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'maccatalyst'">13.1</SupportedOSPlatformVersion>
+    <SupportedOSPlatformVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">21.0</SupportedOSPlatformVersion>
+    <SupportedOSPlatformVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">10.0.17763.0</SupportedOSPlatformVersion>
+    <TargetPlatformMinVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'windows'">10.0.17763.0</TargetPlatformMinVersion>
+    <SupportedOSPlatformVersion Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'tizen'">6.5</SupportedOSPlatformVersion>
+    <NoWarn>$(NoWarn);NU1605</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- App Icon -->
+    <MauiIcon Include="Resources\AppIcon\appicon.svg" ForegroundFile="Resources\AppIcon\appiconfg.svg" Color="#512BD4" />
+
+    <!-- Splash Screen -->
+    <MauiSplashScreen Include="Resources\Splash\splash.svg" Color="#512BD4" BaseSize="128,128" />
+
+    <!-- Images -->
+    <MauiImage Include="Resources\Images\*" />
+    <MauiImage Update="Resources\Images\dotnet_bot.svg" BaseSize="168,208" />
+
+    <!-- Custom Fonts -->
+    <MauiFont Include="Resources\Fonts\*" />
+
+    <!-- Raw Assets (also remove the "Resources\Raw" prefix) -->
+    <MauiAsset Include="Resources\Raw\**" LogicalName="%(RecursiveDir)%(Filename)%(Extension)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
+    <PackageReference Include="DeviceRunners.VisualRunners.Maui" Version="0.1.0-preview.2" />
+    <PackageReference Include="DeviceRunners.VisualRunners.Xunit" Version="0.1.0-preview.2" />
+    <PackageReference Include="DeviceRunners.XHarness.Maui" Version="0.1.0-preview.2" />
+    <PackageReference Include="DeviceRunners.XHarness.Xunit" Version="0.1.0-preview.2" />
+    <PackageReference Include="DeviceRunners.UITesting.Maui" Version="0.1.0-preview.2" />
+    <PackageReference Include="DeviceRunners.UITesting.Xunit" Version="0.1.0-preview.2" />
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.utility" Version="2.9.0" />
+    <PackageReference Include="Microsoft.Maui.Controls" Version="$(MauiPackageVersion)" />
+    <PackageReference Include="Microsoft.Maui.Controls.Compatibility" Version="$(MauiPackageVersion)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\CommunityToolkit.Maui.Core\CommunityToolkit.Maui.Core.csproj" />
+    <ProjectReference Include="..\CommunityToolkit.Maui\CommunityToolkit.Maui.csproj" />
+  </ItemGroup>
+
+</Project>

src/CommunityToolkit.Maui.DeviceTests/MainPage.xaml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<ContentPage xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+             xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+             x:Class="CommunityToolkit.Maui.DeviceTests.MainPage">
+
+    <ScrollView>
+        <VerticalStackLayout
+            Spacing="25"
+            Padding="30,0"
+            VerticalOptions="Center">
+
+            <Image
+                Source="dotnet_bot.png"
+                SemanticProperties.Description="Cute dot net bot waving hi to you!"
+                HeightRequest="200"
+                HorizontalOptions="Center" />
+
+            <Label
+                Text="Hello, World!"
+                SemanticProperties.HeadingLevel="Level1"
+                FontSize="32"
+                HorizontalOptions="Center" />
+
+            <Label
+                Text="Welcome to .NET Multi-platform App UI"
+                SemanticProperties.HeadingLevel="Level2"
+                SemanticProperties.Description="Welcome to dot net Multi platform App U I"
+                FontSize="18"
+                HorizontalOptions="Center" />
+
+            <Button
+                x:Name="CounterBtn"
+                Text="Click me"
+                SemanticProperties.Hint="Counts the number of times you click"
+                Clicked="OnCounterClicked"
+                HorizontalOptions="Center" />
+
+        </VerticalStackLayout>
+    </ScrollView>
+
+</ContentPage>

src/CommunityToolkit.Maui.DeviceTests/MainPage.xaml.cs

@@ -0,0 +1,28 @@
+namespace CommunityToolkit.Maui.DeviceTests;
+
+public partial class MainPage : ContentPage
+{
+	int count = 0;
+
+	public MainPage()
+	{
+		InitializeComponent();
+	}
+
+	private void OnCounterClicked(object sender, EventArgs e)
+	{
+		count++;
+
+		if (count == 1)
+		{
+			CounterBtn.Text = $"Clicked {count} time";
+		}
+		else
+		{
+			CounterBtn.Text = $"Clicked {count} times";
+		}
+
+		SemanticScreenReader.Announce(CounterBtn.Text);
+	}
+}
+