diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.yaml index a794b608299..0b9c7049c74 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.yaml @@ -12,6 +12,8 @@ environment: environment_variables: {} default_max_seconds: 300 framework_version: net8.0 + # uncomment once we implement iteration mechanism in the ASP.NET command + # iterations: 1 benchmark_settings: benchmark_file: C:\InfraRuns\RunNew_All\Suites\ASPNETBenchmarks\ASPNetBenchmarks.csv # To take a dump: --application.options.dumpType full --application.options.dumpOutput diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Server.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Server.yaml index f0db47c2177..b288267908b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Server.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Server.yaml @@ -23,6 +23,7 @@ microbenchmark_configurations: bdn_arguments: --warmupCount 1 --iterationCount 20 --allStats --outliers DontRemove --keepFiles environment: default_max_seconds: 3000 + iterations: 1 output: cpu_columns: additional_report_metrics: diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Workstation.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Workstation.yaml index 6f493b6c2ee..6d3060956e4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Workstation.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Microbenchmark/Microbenchmarks_Workstation.yaml @@ -23,6 +23,7 @@ microbenchmark_configurations: bdn_arguments: --warmupCount 1 --iterationCount 20 --allStats --outliers DontRemove --keepFiles environment: default_max_seconds: 3000 + iterations: 1 output: cpu_columns: additional_report_metrics: diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs index 989c251d05a..4d188436ddc 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs @@ -19,6 +19,8 @@ public class Environment public Dictionary environment_variables { get; set; } = new(); public uint default_max_seconds { get; set; } = 300; public string framework_version { get; set; } = "net8.0"; + // uncomment once we implement iteration mechanism in the ASP.NET command + // public uint Iterations { get; set; } = 1; } public class BenchmarkSettings diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index 7cfa38fb0aa..6d711aac8d1 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -24,6 +24,7 @@ public sealed class Run : RunBase public class Environment { public uint default_max_seconds { get; set; } = 300; + public uint Iterations { get; set; } = 1; } public sealed class MicrobenchmarkConfigurations diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 111e0f5f8e2..d8336bd48fa 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -137,7 +137,6 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi foreach (var run in configuration.Runs) { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} [/]\n"); string runPath = Path.Combine(configuration.Output.Path, run.Key); // Create the run path directory. @@ -150,45 +149,52 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi (string, string) fileNameAndCommand = MicrobenchmarkCommandBuilder.Build(configuration, run, benchmark, invocationCountFromBaseline); run.Value.Name = run.Key; - // Run The BDN process with the trace collector. - using (Process bdnProcess = new()) + for (int iterationIdx = 0; iterationIdx < configuration.Environment.Iterations; iterationIdx++) { - bdnProcess.StartInfo.FileName = fileNameAndCommand.Item1; - bdnProcess.StartInfo.Arguments = fileNameAndCommand.Item2; - bdnProcess.StartInfo.UseShellExecute = false; - bdnProcess.StartInfo.RedirectStandardError = true; - bdnProcess.StartInfo.RedirectStandardOutput = true; - bdnProcess.StartInfo.CreateNoWindow = true; - - StringBuilder consoleOutput = new(); - StringBuilder consoleError = new(); - - bdnProcess.OutputDataReceived += (s, e) => + // Run The BDN process with the trace collector. + using (Process bdnProcess = new()) { - consoleOutput.AppendLine(e.Data); - - }; - - bdnProcess.ErrorDataReceived += (s, e) => - { - consoleError.AppendLine(e.Data); - }; - - using (TraceCollector traceCollector = new TraceCollector(benchmarkCleanedName, collectType, runPath)) - { - bdnProcess.Start(); - bdnProcess.BeginOutputReadLine(); - bdnProcess.BeginErrorReadLine(); - bdnProcess.WaitForExit((int)configuration.Environment.default_max_seconds * 1000); + bdnProcess.StartInfo.FileName = fileNameAndCommand.Item1; + bdnProcess.StartInfo.Arguments = fileNameAndCommand.Item2; + bdnProcess.StartInfo.UseShellExecute = false; + bdnProcess.StartInfo.RedirectStandardError = true; + bdnProcess.StartInfo.RedirectStandardOutput = true; + bdnProcess.StartInfo.CreateNoWindow = true; + + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - Iteration: {iterationIdx} [/]\n"); + + StringBuilder consoleOutput = new(); + StringBuilder consoleError = new(); + + bdnProcess.OutputDataReceived += (s, e) => + { + consoleOutput.AppendLine(e.Data); + + }; + + bdnProcess.ErrorDataReceived += (s, e) => + { + consoleError.AppendLine(e.Data); + }; + + string key = $"{run.Key}.{benchmarkCleanedName}.{iterationIdx}"; + string traceName = $"{run.Key}.{benchmarkCleanedName}.{iterationIdx}"; + using (TraceCollector traceCollector = new TraceCollector(traceName, collectType, runPath)) + { + bdnProcess.Start(); + bdnProcess.BeginOutputReadLine(); + bdnProcess.BeginErrorReadLine(); + bdnProcess.WaitForExit((int)configuration.Environment.default_max_seconds * 1000); + } + + ProcessExecutionDetails details = new(key: key, + commandlineArgs: $"{fileNameAndCommand.Item1} {fileNameAndCommand.Item2}", + environmentVariables: run.Value.environment_variables, + standardError: consoleError.ToString(), + standardOut: consoleOutput.ToString(), + exitCode: bdnProcess.ExitCode); + executionDetails[key] = details; } - - ProcessExecutionDetails details = new(key: $"{run.Key}_{benchmark}", - commandlineArgs: $"{fileNameAndCommand.Item1} {fileNameAndCommand.Item2}", - environmentVariables: run.Value.environment_variables, - standardError: consoleError.ToString(), - standardOut: consoleOutput.ToString(), - exitCode: bdnProcess.ExitCode); - executionDetails[$"{run.Key}_{benchmark}"] = details; } } }