diff --git a/src/benchmarks/gc/Utility-Notebooks/CommitDetails.ipynb b/src/benchmarks/gc/Utility-Notebooks/CommitDetails.ipynb new file mode 100644 index 00000000000..7610acc968f --- /dev/null +++ b/src/benchmarks/gc/Utility-Notebooks/CommitDetails.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Script To Get Commit Differences Between Two Commits for the GC Authors " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "string pathToRepository = \"\"; // Fill in e.g., @\"C:\\Users\\musharm\\source\\repos\\runtime\";\n", + "string baselineCommit = \"2d7eea2\";\n", + "string comparandCommit = \"9e6ba1f\";\n", + "string outputPath = \"\"; // Fill in e.g., \"C:\\\\Users\\\\musharm\\\\Desktop\\\\output.md\";" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiate Methods" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using System.Diagnostics;\n", + "using System.IO;\n", + "\n", + "const string gitCommand = @\"log --author=\"\"^Maoni Stephens\"\" --author=\"\"^Andrew Au\"\" --author=\"\"^Mark Plesko\"\" --author=\"\"^Mukund Raghav Sharma\"\" --author=\"\"^Manish Godse\"\" --pretty=format:\"\"Date: %ad%nAuthor: %an%nTitle: %s%n\"\"\";\n", + "\n", + "public void SaveMarkdownTable(string repositoryPath, string baselineCommit, string comparandCommit, string outputPath)\n", + "{\n", + " (string output, string error) = RunGitCommand(repositoryPath, baselineCommit, comparandCommit);\n", + " if (!string.IsNullOrEmpty(error))\n", + " {\n", + " Console.WriteLine(\"Failed with error: \" + error);\n", + " return;\n", + " }\n", + "\n", + " string table = CreateMarkdownTable(output);\n", + " SaveMarkdownTable(table, outputPath);\n", + "}\n", + "\n", + "public (string? output, string? error) RunGitCommand(string repositoryPath, string baselineCommit, string comparandCommit)\n", + "{\n", + " ProcessStartInfo startInfo = new ProcessStartInfo();\n", + " startInfo.FileName = \"git\";\n", + " startInfo.Arguments = gitCommand + \" \" + baselineCommit + \"...\" + comparandCommit;\n", + " startInfo.WorkingDirectory = repositoryPath;\n", + " startInfo.RedirectStandardOutput = true;\n", + " startInfo.RedirectStandardError = true;\n", + " startInfo.UseShellExecute = false;\n", + "\n", + " Process process = new Process();\n", + " process.StartInfo = startInfo;\n", + " process.Start();\n", + "\n", + " string output = process.StandardOutput.ReadToEnd();\n", + " string error = process.StandardError.ReadToEnd();\n", + " process.WaitForExit();\n", + "\n", + " return (output, error);\n", + "}\n", + "\n", + "public string CreateMarkdownTable(string details)\n", + "{\n", + " string[] lines = details.Split(\"\\n\\n\");\n", + "\n", + " StringBuilder table = new StringBuilder();\n", + " table.AppendLine(\"| Date | Author | Title | PR |\");\n", + " table.AppendLine(\"|------|--------|-------|----|\");\n", + "\n", + " foreach (string line in lines)\n", + " {\n", + " string[] fields = line.Split(\"\\n\");\n", + "\n", + " string date = fields[0].Replace(\"Date: \", \"\");\n", + " string author = fields[1].Replace(\"Author: \", \"\");\n", + " string title = fields[2].Replace(\"Title: \", \"\");\n", + "\n", + " // Extract number in parentheses from the title\n", + " string pr = \"\";\n", + " int startIndex = title.IndexOf('(');\n", + " int endIndex = title.IndexOf(')');\n", + " if (startIndex != -1 && endIndex != -1)\n", + " {\n", + " pr = title.Substring(startIndex + 1, endIndex - startIndex - 1);\n", + " }\n", + "\n", + " // Create markdown link for PR\n", + " string prLink = $\"[{pr}](https://github.com/dotnet/runtime/pull/{pr.Replace(\"#\", \"\")})\";\n", + "\n", + " table.AppendLine($\"| {date} | {author} | {title} | {prLink} |\");\n", + " }\n", + "\n", + " return table.ToString();\n", + "}\n", + "\n", + "// \n", + "public void SaveMarkdownTable(string content, string path)\n", + "{\n", + " File.WriteAllText(path, content);\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "SaveMarkdownTable(pathToRepository, baselineCommit, comparandCommit, outputPath);" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
16056
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "

.NET Interactive

© 2020 Microsoft Corporation

Version: 1.0.522904+cdfa48b2ea1a27dfe0f545c42a34fd3ec7119074

Library version: 1.0.0-beta.24229.4+cdfa48b2ea1a27dfe0f545c42a34fd3ec7119074

Build date: 2024-05-01T22:47:43.3565220Z

https://github.com/dotnet/interactive

" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "System.Diagnostics.Process.GetCurrentProcess().Id\n", + "#!about" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "languageName": "csharp", + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/benchmarks/gc/Utility-Notebooks/DumpsAnalyzer.ipynb b/src/benchmarks/gc/Utility-Notebooks/DumpsAnalyzer.ipynb new file mode 100644 index 00000000000..64954bc94c6 --- /dev/null +++ b/src/benchmarks/gc/Utility-Notebooks/DumpsAnalyzer.ipynb @@ -0,0 +1,310 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Script to extract and aggregate call stack infomation for dumps in given folder." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "using System.Diagnostics;\n", + "using System.IO;\n", + "using System.Text.Json;\n", + "using System.Text.RegularExpressions;\n", + "\n", + "\n", + "string windbgPath = \"C:\\\\Users\\\\v-bibu\\\\windbg\\\\windbg.exe\";\n", + "string dumpFolder = @\"Q:\\dumps-testhang\";\n", + "List debugCommandList = new List(){\n", + " \".symfix\",\n", + " \"k\",\n", + " \"q\"\n", + "};" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyze class" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "\n", + "public class DumpAnalyzer\n", + "{\n", + " #nullable enable\n", + " private string? windbgPath;\n", + "\n", + " public DumpAnalyzer(string windbgPath) \n", + " {\n", + " if (String.IsNullOrEmpty(windbgPath))\n", + " throw new ArgumentException($\"Path to windbg.exe should be provided.\");\n", + "\n", + " if (!Path.Exists(windbgPath))\n", + " throw new ArgumentException($\"The given windbg path {windbgPath} doesn't exist.\");\n", + "\n", + " this.windbgPath = windbgPath;\n", + " }\n", + "\n", + " public void DebugDumpWithWinDBG(List debugCommandList, string dumpPath, string outputPath)\n", + " {\n", + " var debugCommandSequence = debugCommandList\n", + " .Select(command => $\"{command}\\n\")\n", + " .Aggregate(String.Concat);\n", + "\n", + " try\n", + " {\n", + " Console.WriteLine($\"Start to debug dump: {dumpPath}.\");\n", + " using (Process process = new Process())\n", + " {\n", + " process.StartInfo.FileName = windbgPath;\n", + " process.StartInfo.Arguments = $\"-z {dumpPath} -c \\\"{debugCommandSequence}\\\" -logo {outputPath}\";\n", + " process.StartInfo.UseShellExecute = false;\n", + " process.Start();\n", + "\n", + " process.WaitForExit();\n", + " }\n", + " Console.WriteLine($\"Write debug output to: {outputPath}.\");\n", + " }\n", + " catch (Exception ex)\n", + " {\n", + " Console.WriteLine(ex.Message);\n", + " }\n", + " }\n", + "\n", + " public string? ExtractCallStackFromDebugOutput(string debugOutput)\n", + " {\n", + " string pattern = @\"# Child-SP\\s+RetAddr\\s+Call Site\\s+(.*?)\\d+:\\d+>\\s.?\";\n", + " Match match = Regex.Match(debugOutput, pattern, RegexOptions.Singleline);\n", + " if (match.Success)\n", + " {\n", + " return match.Groups[1].Value.Trim();\n", + " }\n", + " else\n", + " {\n", + " Console.WriteLine($\"can't extract call stack info from debug output.\");\n", + " return null;\n", + " }\n", + " }\n", + "\n", + " public void SaveCallStack(List debugCommandList, string dumpFolder)\n", + " {\n", + " foreach (string dumpPath in Directory.GetFiles(dumpFolder, \"stress*.dmp\"))\n", + " {\n", + " Console.WriteLine($\"====== Debugging {dumpPath} and extracting call stack ======\");\n", + " try\n", + " {\n", + " string outputPath = dumpPath.Replace(\".dmp\", \".txt\");\n", + " DebugDumpWithWinDBG(debugCommandList, dumpPath, outputPath);\n", + "\n", + " string debugOutput = File.ReadAllText(outputPath);\n", + " string? callStackInfo = ExtractCallStackFromDebugOutput(debugOutput);\n", + " if (string.IsNullOrEmpty(callStackInfo))\n", + " continue;\n", + "\n", + " File.WriteAllText(outputPath, callStackInfo);\n", + " }\n", + " catch (Exception ex)\n", + " {\n", + " Console.WriteLine($\"fail to save call stack for {dumpPath}: {ex.Message}\");\n", + " }\n", + " }\n", + " }\n", + "\n", + " public (string?, int?) GetFileNameAndLineNumberOfTopStackFrame(string keyword, string callStack)\n", + " {\n", + " string[] lines = callStack.Split(new[] { '\\r', '\\n' });\n", + " string lineContainKeyword = lines.FirstOrDefault(line => line.Contains(keyword), String.Empty);\n", + "\n", + " if (String.IsNullOrEmpty(lineContainKeyword)) \n", + " {\n", + " Console.WriteLine($\"Fail to find the keyword in call stack.\");\n", + " return (null, null);\n", + " }\n", + "\n", + " // look for the pattern [filename@linenumber]\n", + " string pattern = @\"\\[(.*?)\\]\";\n", + " Match match = Regex.Match(lineContainKeyword, pattern, RegexOptions.Singleline);\n", + "\n", + " if (!match.Success)\n", + " {\n", + " Console.WriteLine($\"The symbol is not available.\");\n", + " return (null, null);\n", + " }\n", + " \n", + " string fileNameWithLineNumber = match.Groups[1].Value.Trim();\n", + " string[] splitOutput = fileNameWithLineNumber.Split(\"@\");\n", + "\n", + " string? fileName = splitOutput.FirstOrDefault(String.Empty);\n", + " if (String.IsNullOrEmpty(fileName))\n", + " {\n", + " Console.WriteLine($\"Fail to extract filename.\");\n", + " fileName = null;\n", + " }\n", + "\n", + " string? lineNumberstr = splitOutput.LastOrDefault(String.Empty).Trim();\n", + " if (String.IsNullOrEmpty(lineNumberstr))\n", + " {\n", + " Console.WriteLine($\"Fail to extract line number.\");\n", + " return (fileName, null);\n", + " }\n", + "\n", + " bool success = int.TryParse(lineNumberstr, out int lineNumber);\n", + " if (!success)\n", + " {\n", + " Console.WriteLine($\"Fail to parse line number.\");\n", + " return (fileName, null);\n", + " }\n", + "\n", + " return (fileName, lineNumber);\n", + "\n", + " }\n", + "\n", + " public void GetAndSaveKey(string keyword, string debugOutputFolder)\n", + " {\n", + " string[] callStackFileList = Directory.GetFiles(debugOutputFolder, \"stress*.txt\");\n", + " Dictionary> keyToPath = new();\n", + "\n", + "\n", + " foreach (string callStackFile in callStackFileList)\n", + " {\n", + " Console.WriteLine($\"====== Processing {callStackFile} ======\");\n", + "\n", + " try\n", + " {\n", + " string callStackInfo = File.ReadAllText(callStackFile);\n", + "\n", + " (string? fileName, int? lineNumber) = GetFileNameAndLineNumberOfTopStackFrame(keyword, callStackInfo);\n", + "\n", + " if (fileName == null || lineNumber == null) \n", + " {\n", + " if (!keyToPath.TryGetValue(\"unknown\", out var unknownKVP))\n", + " {\n", + " keyToPath[\"unknown\"] = new List();\n", + " }\n", + " keyToPath[\"unknown\"].Add(callStackFile);\n", + " continue;\n", + " }\n", + "\n", + " string srcLine = File.ReadAllLines(fileName)[lineNumber.Value - 1];\n", + "\n", + " if (!keyToPath.TryGetValue(srcLine, out var kvp))\n", + " {\n", + " keyToPath[srcLine] = new List();\n", + " }\n", + "\n", + " keyToPath[srcLine].Add(callStackFile);\n", + " }\n", + "\n", + " catch (Exception ex)\n", + " {\n", + " Console.WriteLine(ex.Message);\n", + " }\n", + " }\n", + "\n", + " string jsonString = JsonSerializer.Serialize(keyToPath, new JsonSerializerOptions\n", + " {\n", + " WriteIndented = true // Makes the JSON output more readable with indentation\n", + " });\n", + "\n", + " // Write JSON string to a file\n", + " string filePath = Path.Combine(debugOutputFolder, \"dictionary.json\");\n", + " File.WriteAllText(filePath, jsonString);\n", + " }\n", + "}\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "====== Debugging Q:\\dumps-testhang\\stress_1e68_2024-07-16_01-51-14-392_44bc.dmp and extracting call stack ======\n", + "Start to debug dump: Q:\\dumps-testhang\\stress_1e68_2024-07-16_01-51-14-392_44bc.dmp.\n", + "Write debug output to: Q:\\dumps-testhang\\stress_1e68_2024-07-16_01-51-14-392_44bc.txt.\n", + "====== Processing Q:\\dumps-testhang\\stress_1e68_2024-07-16_01-51-14-392_44bc.txt ======\n", + "Fail to find the keyword in call stack.\n" + ] + } + ], + "source": [ + "var analyzer = new DumpAnalyzer(windbgPath);\n", + "analyzer.SaveCallStack(debugCommandList, dumpFolder);\n", + "analyzer.GetAndSaveKey(\"::gc_heap::\", dumpFolder);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "languageName": "csharp", + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/benchmarks/gc/Utility-Notebooks/FindDotnetRuntimeProvider.ipynb b/src/benchmarks/gc/Utility-Notebooks/FindDotnetRuntimeProvider.ipynb new file mode 100644 index 00000000000..396a8ec57e2 --- /dev/null +++ b/src/benchmarks/gc/Utility-Notebooks/FindDotnetRuntimeProvider.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Script to Find Out Which Sessions Enable The DotnetRunTime Provider " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "using System;\n", + "using System.Collections.Generic;\n", + "using System.Diagnostics;\n", + "using System.Text.RegularExpressions;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "public sealed class EventTraceProvider\n", + "{\n", + " public string Name { get; }\n", + " public string Guid { get; }\n", + " public string Type { get; }\n", + " public string Status { get; }\n", + "\n", + " public EventTraceProvider(string name, string guid, string type, string status)\n", + " {\n", + " Name = name;\n", + " Guid = guid;\n", + " Type = type;\n", + " Status = status;\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + } + }, + "outputs": [], + "source": [ + "public static void Run()\n", + "{\n", + " string logmanOutput = RunLogmanCommand(\"query\", \"-ets\");\n", + "\n", + " // Parse and display the data\n", + " List providers = ParseLogmanOutput(logmanOutput);\n", + "\n", + " foreach (EventTraceProvider provider in providers)\n", + " {\n", + " //if (provider.Name.StartsWith(\"PerfView\") || provider.Name.StartsWith(\"AppModel\"))\n", + " {\n", + "\n", + " Console.WriteLine($\"Provider Name: {provider.Name}\");\n", + "\n", + " // Retrieve and parse details for the provider\n", + " string providerDetailsOutput = RunLogmanCommand(\"query\", \"-ets -n \" + provider.Name);\n", + " //Dictionary providerDetails = ParseProviderDetails(providerDetailsOutput, args[0]);\n", + " string providerDetails = ParseProviderDetails(providerDetailsOutput);\n", + " if (providerDetails != null) \n", + " {\n", + " Console.WriteLine(providerDetails);\n", + " }\n", + "\n", + " Console.WriteLine();\n", + " }\n", + " }\n", + "\n", + "}\n", + "\n", + "public static string RunLogmanCommand(string action, string arguments)\n", + "{\n", + " try\n", + " {\n", + " // Create process start info\n", + " ProcessStartInfo psi = new ProcessStartInfo();\n", + " psi.FileName = \"logman\";\n", + " psi.Arguments = $\"{action} {arguments}\";\n", + " psi.RedirectStandardOutput = true;\n", + " psi.UseShellExecute = false;\n", + "\n", + " // Start the process\n", + " Process process = Process.Start(psi);\n", + "\n", + " // Read the output\n", + " string output = process.StandardOutput.ReadToEnd();\n", + "\n", + " // Close the process\n", + " process.WaitForExit();\n", + " process.Close();\n", + "\n", + " return output;\n", + " }\n", + " catch (Exception ex)\n", + " {\n", + " Console.WriteLine(\"Error: \" + ex.Message);\n", + " return null;\n", + " }\n", + "}\n", + "\n", + "\n", + "public static List ParseLogmanOutput(string logmanOutput)\n", + "{\n", + " List providers = new List();\n", + "\n", + " string[] lines = logmanOutput.Split(new[] { Environment.NewLine }, StringSplitOptions.None);\n", + "\n", + " foreach (string line in lines)\n", + " {\n", + " int providerEnd = line.IndexOf(\"Trace Running\");\n", + " if (providerEnd != -1)\n", + " {\n", + " string providerName = line.Substring(0, providerEnd).TrimEnd();\n", + " //Console.WriteLine(\"---{0}------\", providerName);\n", + " providers.Add(new EventTraceProvider(providerName, null, null, null));\n", + " }\n", + " }\n", + "\n", + " return providers;\n", + "}\n", + "\n", + "public static string ParseProviderDetails(string providerDetailsOutput)\n", + "{\n", + " Dictionary details = new Dictionary();\n", + "\n", + " string[] lines = providerDetailsOutput.Split(new[] { Environment.NewLine }, StringSplitOptions.None);\n", + "\n", + " foreach (string line in lines)\n", + " {\n", + " if (line.StartsWith(\"Name: Microsoft-Windows-DotNETRuntime\"))\n", + " {\n", + " return providerDetailsOutput;\n", + " }\n", + " }\n", + "\n", + " return null;\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Run()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "System.Diagnostics.Process.GetCurrentProcess().Id\n", + "#!about" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (C#)", + "language": "C#", + "name": ".net-csharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "csharp", + "items": [ + { + "aliases": [], + "languageName": "csharp", + "name": "csharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}