[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -44,14 +44,14 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24408.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24516.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>60ae233c3d77f11c5fdb53e570b64d503b13ba59</Sha>
+      <Sha>3c393bbd85ae16ddddba20d0b75035b0c6f1a52d</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24408.2">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24516.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>60ae233c3d77f11c5fdb53e570b64d503b13ba59</Sha>
+      <Sha>3c393bbd85ae16ddddba20d0b75035b0c6f1a52d</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DiaSymReader.Pdb2Pdb" Version="1.1.0-beta2-19575-01">
@@ -62,9 +62,9 @@
       <Uri>https://github.com/dotnet/symreader-converter</Uri>
       <Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24408.2">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24516.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>60ae233c3d77f11c5fdb53e570b64d503b13ba59</Sha>
+      <Sha>3c393bbd85ae16ddddba20d0b75035b0c6f1a52d</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/SetupNugetSources.ps1

@@ -157,7 +157,7 @@ if ($dotnet31Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
-$dotnetVersions = @('5','6','7','8')
+$dotnetVersions = @('5','6','7','8','9')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;

eng/common/SetupNugetSources.sh

@@ -99,7 +99,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7' '8')
+DotNetVersions=('5' '6' '7' '8' '9')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";

eng/common/core-templates/job/job.yml

@@ -33,11 +33,6 @@ parameters:
   artifactPublishSteps: []
   runAsPublic: false
 
-# Sbom related params
-  enableSbom: true
-  PackageVersion: 9.0.0
-  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
-
 # 1es specific parameters
   is1ESPipeline: ''
 

eng/common/core-templates/steps/get-delegation-sas.yml

@@ -31,7 +31,16 @@ steps:
       # Calculate the expiration of the SAS token and convert to UTC
       $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
 
-      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+      # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+      # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+      $sas = ""
+      do {
+        $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+        if ($LASTEXITCODE -ne 0) {
+          Write-Error "Failed to generate SAS token."
+          exit 1
+        }
+      } while($sas.IndexOf('/') -ne -1)
 
       if ($LASTEXITCODE -ne 0) {
         Write-Error "Failed to generate SAS token."

eng/common/internal/Tools.csproj

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
+    <BuildWithNetFrameworkHostedCompiler>false</BuildWithNetFrameworkHostedCompiler>
   </PropertyGroup>
   <ItemGroup>
     <!-- Clear references, the SDK may add some depending on UsuingToolXxx settings, but we only want to restore the following -->

eng/common/templates-official/job/job.yml

@@ -1,8 +1,23 @@
+parameters:
+# Sbom related params
+  enableSbom: true
+  runAsPublic: false
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+
 jobs:
 - template: /eng/common/core-templates/job/job.yml
   parameters:
     is1ESPipeline: true
 
+    componentGovernanceSteps:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+      - template: /eng/common/templates/steps/generate-sbom.yml
+        parameters:
+          PackageVersion: ${{ parameters.packageVersion }}
+          BuildDropPath: ${{ parameters.buildDropPath }}
+          publishArtifacts: false
+
     # publish artifacts
     # for 1ES managed templates, use the templateContext.output to handle multiple outputs.
     templateContext:

eng/common/templates/job/job.yml

@@ -4,6 +4,7 @@ parameters:
   componentGovernanceIgnoreDirectories: ''
 # Sbom related params
   enableSbom: true
+  runAsPublic: false
   PackageVersion: 9.0.0
   BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
 
@@ -19,71 +20,63 @@ jobs:
     steps:
     - ${{ each step in parameters.steps }}:
       - ${{ step }}
-    
+
     componentGovernanceSteps:
-      - template: /eng/common/templates/steps/component-governance.yml
-        parameters:
-          ${{ if eq(parameters.disableComponentGovernance, '') }}:
-            ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-              disableComponentGovernance: false
-            ${{ else }}:
-              disableComponentGovernance: true
+    - template: /eng/common/templates/steps/component-governance.yml
+      parameters:
+        ${{ if eq(parameters.disableComponentGovernance, '') }}:
+          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+            disableComponentGovernance: false
           ${{ else }}:
-            disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
-          componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
-
-      - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
-        - template: /eng/common/templates/steps/generate-sbom.yml
-          parameters:
-            PackageVersion: ${{ parameters.packageVersion }}
-            BuildDropPath: ${{ parameters.buildDropPath }}
-            publishArtifacts: false
-
+            disableComponentGovernance: true
+        ${{ else }}:
+          disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
+        componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
 
     artifactPublishSteps:
-      - ${{ if ne(parameters.artifacts.publish, '') }}:
-        - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
-          - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
-            parameters:
-              is1ESPipeline: false
-              args:
-                displayName: Publish pipeline artifacts
-                pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
-                publishLocation: Container
-                artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
-                continueOnError: true
-                condition: always()
-        - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
-          - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
-            parameters:
-              is1ESPipeline: false
-              args:
-                targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log'
-                artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
-                displayName: 'Publish logs'
-                continueOnError: true
-                condition: always()
-                sbomEnabled: false  # we don't need SBOM for logs
-
-      - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
+    - ${{ if ne(parameters.artifacts.publish, '') }}:
+      - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
         - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
           parameters:
             is1ESPipeline: false
             args:
-              displayName: Publish Logs
-              pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+              displayName: Publish pipeline artifacts
+              pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
               publishLocation: Container
-              artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+              artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
               continueOnError: true
               condition: always()
-
-      - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
+      - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
         - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
           parameters:
             is1ESPipeline: false
             args:
-              targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration'
-              artifactName: 'BuildConfiguration'
-              displayName: 'Publish build retry configuration'
+              targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log'
+              artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+              displayName: 'Publish logs'
               continueOnError: true
-              sbomEnabled: false  # we don't need SBOM for BuildConfiguration
+              condition: always()
+              sbomEnabled: false  # we don't need SBOM for logs
+
+    - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
+      - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+        parameters:
+          is1ESPipeline: false
+          args:
+            displayName: Publish Logs
+            pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+            publishLocation: Container
+            artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+            continueOnError: true
+            condition: always()
+
+    - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
+      - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+        parameters:
+          is1ESPipeline: false
+          args:
+            targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration'
+            artifactName: 'BuildConfiguration'
+            displayName: 'Publish build retry configuration'
+            continueOnError: true
+            sbomEnabled: false  # we don't need SBOM for BuildConfiguration

eng/common/tools.ps1

@@ -900,7 +900,7 @@ function IsWindowsPlatform() {
 }
 
 function Get-Darc($version) {
-  $darcPath  = "$TempDir\darc\$(New-Guid)"
+  $darcPath  = "$TempDir\darc\$([guid]::NewGuid())"
   if ($version -ne $null) {
     & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
   } else {

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "9.0.100-preview.5.24307.3",
+    "version": "9.0.100-rc.2.24474.11",
     "rollForward": "minor",
     "allowPrerelease": false,
     "architecture": "x64"
@@ -29,9 +29,9 @@
     },
     "xcopy-msbuild": "17.8.5",
     "vswhere": "2.2.7",
-    "dotnet": "9.0.100-preview.5.24307.3"
+    "dotnet": "9.0.100-rc.2.24474.11"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24408.2"
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24516.2"
   }
 }

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs

@@ -23,12 +23,8 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
         var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj");
-        InvokeDotnetTest($@"{projectPath} -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", new Dictionary<string, string?>
-        {
-            // Setting this temporarily, until we upgrade to final net9, which has this option set automatically in the MSBUILD sdk.
-            // Without this option we don't produce any output to the terminal logger.
-            ["_MSBUILDTLENABLED"] = "1"
-        });
+        // Forcing terminal logger so we can see the output when it is redirected
+        InvokeDotnetTest($@"{projectPath} -tl:on -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}");
 
         // The output:
         // Determining projects to restore...
@@ -38,7 +34,8 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru
         // C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\UnitTest1.cs(41): error VSTEST1: (FailingTest) SampleUnitTestProject.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<2>. Actual:<3>.  [C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\SimpleTestProject.csproj::TargetFramework=net462]
         // C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\UnitTest1.cs(41): error VSTEST1: (FailingTest) SampleUnitTestProject.UnitTest1.FailingTest() Assert.AreEqual failed. Expected:<2>. Actual:<3>.  [C:\Users\nohwnd\AppData\Local\Temp\vstest\xvoVt\SimpleTestProject.csproj::TargetFramework=netcoreapp3.1]
 
-        StdOutputContains("error TESTERROR: FailingTest (");
+        StdOutputContains("TESTERROR");
+        StdOutputContains("FailingTest (");
         StdOutputContains("): Error Message: Assert.AreEqual failed. Expected:<ğğğ𦮙我們剛才從𓋴𓅓𓏏𓇏𓇌𓀀>. Actual:<not the same>.");
         StdOutputContains("at TerminalLoggerUnitTests.UnitTest1.FailingTest() in");
         // We are sending those as low prio messages, they won't show up on screen but will be in binlog.

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs

@@ -862,7 +862,7 @@ protected static void ExecuteApplication(string path, string? args, out string s
         process.Start();
         process.BeginOutputReadLine();
         process.BeginErrorReadLine();
-        if (!process.WaitForExit(5 * 60 * 1000)) // 5 minutes
+        if (!process.WaitForExit(30 * 60 * 1000)) // 30 minutes
         {
             Console.WriteLine($"IntegrationTestBase.Execute: Timed out waiting for {executableName}. Terminating the process.");
             process.Kill();