From ad35a7ca768a651981361d4ab6f1a29e939f7a8c Mon Sep 17 00:00:00 2001 From: Jatin Garg <48029724+jatgarg@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:23:36 -0700 Subject: [PATCH] Add code-coverage-tools package to compare code coverage on the PR build. (#22452) ## Description [AB#14170](https://dev.azure.com/fluidframework/internal/_workitems/edit/14170) Add code-coverage-tools package to compare code coverage on the PR build. 1.) Before running coverage comparison, code coverage plugin identifies the baseline build for the PR. 2.) Once the baseline build is identified, we download the build artifacts corresponding to the `Code Coverage Report_` artifact name for this build 3.) We then collect the code coverage stats for the PR build and then make the comparison with the baseline. 4.) If the code coverage diff (branch coverage) is more than a percentage point change, then we fail the build for the PR. We also fail the build in case the code coverage for the newly added package is less than 50%. 5.) We post the comment on the PR specifying the code coverage change if any for each package which is modified. We needed this separate module as we need specifics to do the code coverage comparison like the baseline with which we need to make the comparison and then what we need to compare and then after comparison what comment and in what format we want to report it. Sample Comment: code coverage --------- Co-authored-by: Jatin Garg Co-authored-by: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Co-authored-by: Tyler Butler --- build-tools/packages/build-cli/.eslintrc.cjs | 4 + build-tools/packages/build-cli/README.md | 1 + .../build-cli/docs/codeCoverageDetails.md | 93 ++++++++ build-tools/packages/build-cli/docs/report.md | 41 ++++ build-tools/packages/build-cli/package.json | 9 +- .../src/codeCoverage/codeCoveragePr.ts | 86 +++++++ .../src/codeCoverage/compareCodeCoverage.ts | 213 ++++++++++++++++++ .../codeCoverage/getCommentForCodeCoverage.ts | 128 +++++++++++ .../src/codeCoverage/getCoverageMetrics.ts | 114 ++++++++++ .../src/commands/report/codeCoverage.ts | 165 ++++++++++++++ .../src/library/azureDevops/constants.ts | 43 ++++ .../azureDevops/getBaselineBuildMetrics.ts | 189 ++++++++++++++++ .../src/library/azureDevops/utils.ts | 82 +++++++ .../build-cli/src/library/githubRest.ts | 77 +++++++ build-tools/pnpm-lock.yaml | 51 ++++- 15 files changed, 1285 insertions(+), 11 deletions(-) create mode 100644 build-tools/packages/build-cli/docs/codeCoverageDetails.md create mode 100644 build-tools/packages/build-cli/docs/report.md create mode 100644 build-tools/packages/build-cli/src/codeCoverage/codeCoveragePr.ts create mode 100644 build-tools/packages/build-cli/src/codeCoverage/compareCodeCoverage.ts create mode 100644 build-tools/packages/build-cli/src/codeCoverage/getCommentForCodeCoverage.ts create mode 100644 build-tools/packages/build-cli/src/codeCoverage/getCoverageMetrics.ts create mode 100644 build-tools/packages/build-cli/src/commands/report/codeCoverage.ts create mode 100644 build-tools/packages/build-cli/src/library/azureDevops/constants.ts create mode 100644 build-tools/packages/build-cli/src/library/azureDevops/getBaselineBuildMetrics.ts create mode 100644 build-tools/packages/build-cli/src/library/azureDevops/utils.ts diff --git a/build-tools/packages/build-cli/.eslintrc.cjs b/build-tools/packages/build-cli/.eslintrc.cjs index b154bc3f8d5e..80f9475e5dfe 100644 --- a/build-tools/packages/build-cli/.eslintrc.cjs +++ b/build-tools/packages/build-cli/.eslintrc.cjs @@ -38,9 +38,13 @@ module.exports = { // These are all excluded because they're "submodules" used for organization. // AB#8118 tracks removing the barrel files and importing directly from the submodules. "**/library/index.js", + "**/library/githubRest.js", "**/handlers/index.js", "**/machines/index.js", "**/repoPolicyCheck/index.js", + "**/azureDevops/**", + "**/codeCoverage/**", + "azure-devops-node-api/**", ], }, ], diff --git a/build-tools/packages/build-cli/README.md b/build-tools/packages/build-cli/README.md index 16a4fe4e57ad..d7f6bdc69b9a 100644 --- a/build-tools/packages/build-cli/README.md +++ b/build-tools/packages/build-cli/README.md @@ -47,6 +47,7 @@ USAGE * [`flub publish`](docs/publish.md) - Publish commands are used to publish packages to an npm registry. * [`flub release`](docs/release.md) - Release commands are used to manage the Fluid release process. * [`flub rename-types`](docs/rename-types.md) - Renames type declaration files from .d.ts to .d.mts. +* [`flub report`](docs/report.md) - Report analysis about the codebase, like code coverage and bundle size measurements. * [`flub run`](docs/run.md) - Generate a report from input bundle stats collected through the collect bundleStats command. * [`flub transform`](docs/transform.md) - Transform commands are used to transform code, docs, etc. into alternative forms. * [`flub typetests`](docs/typetests.md) - Updates configuration for type tests in package.json files. If the previous version changes after running preparation, then npm install must be run before building. diff --git a/build-tools/packages/build-cli/docs/codeCoverageDetails.md b/build-tools/packages/build-cli/docs/codeCoverageDetails.md new file mode 100644 index 000000000000..43b2d8564d14 --- /dev/null +++ b/build-tools/packages/build-cli/docs/codeCoverageDetails.md @@ -0,0 +1,93 @@ +# Code coverage + +## Overview + +This module contains all the utilities required to analyze code coverage metrics from PRs. The coverage metrics generated in the PR build are compared against a baseline CI build for packages that have been updated in the PR. If the line or branch coverage for a package has been impacted in the PR, a comment is posted to the PR showing the diff of the code coverage between baseline and PR. + +## Code Coverage Metrics + +Code coverage metrics is generated when tests run in the CI pipeline. You can also generate the code coverage metrics for a package locally by running `npm run test:coverage` for the individual package or by running `npm run ci:test:mocha:coverage` from the root. + +## Pieces of the code coverage analysis + +Code coverage has several steps involving different commands. This section defines those pieces and how they fit together to enable overall code coverage reporting. + +### Cobertura coverage files + +Code coverage metrics is included in the cobertura-format coverage files we collect during CI builds. These files are currently published as artifacts from both our PR and internal build pipelines to ensure we can run comparisons on PRs against a baseline build. + +### Identifying the baseline build + +Before running coverage comparison, a baseline build needs to be determined for the PR. This is typically based on the target branch for the PR. For example, if a pull request was targeting `main`, we would consider the baseline to be the latest successful `main` branch build. + +### Downloading artifacts from baseline build + +Once the baseline build is identified, we download the build artifacts corresponding to the `Code Coverage Report_{Build_Number}` artifact name for this build. We unzip the files and extract the coverage metrics out of the code coverage artifact using the helper `getCoverageMetricsFromArtifact`. Currently, we track `lineCoverage` and `branchCoverage` as our metrics for reporting and +use both `lineCoverage` and `branchCoverage` for success criteria. The metrics is in percentages from 0..100. The final structure of the extracted metrics looks like the following: + +```typescript +/** + * The type for the coverage report, containing the line coverage and branch coverage(in percentage) for each package + */ +export interface CoverageMetric { + lineCoverage: number; + branchCoverage: number; +} +``` + +### Generating the coverage metrics on PR build + +As mentioned earlier, the PR build also uploads coverage metrics as artifacts that can be used to run coverage analysis against a baseline build. To help with this, we use the `getCoverageMetricsFromArtifact` helper function to extract code coverage metrics of format `CoverageMetric` that contains code coverage metrics corresponding to the PR for each package. + +### Comparing code coverage reports + +Once we have the coverage metrics for the baseline and PR build, we use the `compareCodeCoverage` utility function that returns an array of coverage comparisons for the list of packages passed into it. The array returned contains objects of type `CodeCoverageComparison`. We ignore some packages for comparing code coverage such as definition packages and packages which are not inside `packages` folder in the repo. The logic is in the `compareCodeCoverage` function. + +```typescript +/** + * Type for the code coverage report generated by comparing the baseline and pr code coverage + */ +export interface CodeCoverageComparison { + /** + * Path of the package + */ + packagePath: string; + /** + * Line coverage in baseline build (as a percent) + */ + lineCoverageInBaseline: number; + /** + * Line coverage in pr build (as a percent) + */ + lineCoverageInPr: number; + /** + * difference between line coverage in pr build and baseline build (percentage points) + */ + lineCoverageDiff: number; + /** + * branch coverage in baseline build (as a percent) + */ + branchCoverageInBaseline: number; + /** + * branch coverage in pr build (as a percent) + */ + branchCoverageInPr: number; + /** + * difference between branch coverage in pr build and baseline build (percentage points) + */ + branchCoverageDiff: number; + /** + * Flag to indicate if the package is new + */ + isNewPackage: boolean; +} +``` + +### Success Criteria + +The code coverage PR checks will fail in the following cases: + +- If the **line code coverage** or **branch code coverage** decreased by > 1%. +- If the code coverage(line and branch) for a newly added package is < 50%. + +The enforcement code is in the function `isCodeCoverageCriteriaPassed`. diff --git a/build-tools/packages/build-cli/docs/report.md b/build-tools/packages/build-cli/docs/report.md new file mode 100644 index 000000000000..12ad7d3f5e07 --- /dev/null +++ b/build-tools/packages/build-cli/docs/report.md @@ -0,0 +1,41 @@ +`flub report` +============= + +Report analysis about the codebase, like code coverage and bundle size measurements. + +* [`flub report codeCoverage`](#flub-report-codecoverage) + +## `flub report codeCoverage` + +Run comparison of code coverage stats + +``` +USAGE + $ flub report codeCoverage --adoBuildId --adoApiToken --githubApiToken + --adoCIBuildDefinitionIdBaseline --adoCIBuildDefinitionIdPR + --codeCoverageAnalysisArtifactNameBaseline --codeCoverageAnalysisArtifactNamePR --githubPRNumber + --githubRepositoryName --githubRepositoryOwner --targetBranchName [-v | --quiet] + +FLAGS + --adoApiToken= (required) Token to get auth for accessing ADO builds. + --adoBuildId= (required) Azure DevOps build ID. + --adoCIBuildDefinitionIdBaseline= (required) Build definition/pipeline number/id for the baseline + build. + --adoCIBuildDefinitionIdPR= (required) Build definition/pipeline number/id for the PR build. + --codeCoverageAnalysisArtifactNameBaseline= (required) Code coverage artifact name for the baseline build. + --codeCoverageAnalysisArtifactNamePR= (required) Code coverage artifact name for the PR build. + --githubApiToken= (required) Token to get auth for accessing Github PR. + --githubPRNumber= (required) Github PR number. + --githubRepositoryName= (required) Github repository name. + --githubRepositoryOwner= (required) Github repository owner. + --targetBranchName= (required) Target branch name. + +LOGGING FLAGS + -v, --verbose Enable verbose logging. + --quiet Disable all logging. + +DESCRIPTION + Run comparison of code coverage stats +``` + +_See code: [src/commands/report/codeCoverage.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/report/codeCoverage.ts)_ diff --git a/build-tools/packages/build-cli/package.json b/build-tools/packages/build-cli/package.json index 5f7124b39db4..e06349015f8d 100644 --- a/build-tools/packages/build-cli/package.json +++ b/build-tools/packages/build-cli/package.json @@ -93,6 +93,7 @@ "@octokit/rest": "^21.0.2", "@rushstack/node-core-library": "^3.59.5", "async": "^3.2.4", + "azure-devops-node-api": "^11.2.0", "chalk": "^5.3.0", "change-case": "^3.1.0", "cosmiconfig": "^8.3.6", @@ -110,6 +111,7 @@ "issue-parser": "^7.0.1", "json5": "^2.2.3", "jssm": "5.98.2", + "jszip": "^3.10.1", "latest-version": "^5.1.0", "mdast": "^3.0.0", "mdast-util-heading-range": "^4.0.0", @@ -137,7 +139,8 @@ "table": "^6.8.1", "ts-morph": "^22.0.0", "type-fest": "^2.19.0", - "unist-util-visit": "^5.0.0" + "unist-util-visit": "^5.0.0", + "xml2js": "^0.5.0" }, "devDependencies": { "@biomejs/biome": "~1.8.3", @@ -161,6 +164,7 @@ "@types/semver-utils": "^1.1.1", "@types/sort-json": "^2.0.1", "@types/unist": "^3.0.3", + "@types/xml2js": "^0.4.11", "c8": "^7.14.0", "chai": "^4.3.7", "chai-arrays": "^2.2.0", @@ -243,6 +247,9 @@ }, "transform": { "description": "Transform commands are used to transform code, docs, etc. into alternative forms." + }, + "report": { + "description": "Report analysis about the codebase, like code coverage and bundle size measurements." } } } diff --git a/build-tools/packages/build-cli/src/codeCoverage/codeCoveragePr.ts b/build-tools/packages/build-cli/src/codeCoverage/codeCoveragePr.ts new file mode 100644 index 000000000000..6826d574813c --- /dev/null +++ b/build-tools/packages/build-cli/src/codeCoverage/codeCoveragePr.ts @@ -0,0 +1,86 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { getAzureDevopsApi } from "@fluidframework/bundle-size-tools"; +import { type IAzureDevopsBuildCoverageConstants } from "../library/azureDevops/constants.js"; +import { + type IBuildMetrics, + getBaselineBuildMetrics, + getBuildArtifactForSpecificBuild, +} from "../library/azureDevops/getBaselineBuildMetrics.js"; +import type { CommandLogger } from "../logging.js"; +import { type CodeCoverageComparison, compareCodeCoverage } from "./compareCodeCoverage.js"; +import { getCoverageMetricsFromArtifact } from "./getCoverageMetrics.js"; + +/** + * Report of code coverage comparison. + */ +export interface CodeCoverageReport { + /** + * Comparison data for each package. + */ + comparisonData: CodeCoverageComparison[]; + + /** + * Baseline build metrics against which the PR build metrics are compared. + */ + baselineBuildMetrics: IBuildMetrics; +} + +/** + * API to get the code coverage report for a PR. + * @param adoToken - ADO token that will be used to download artifacts from ADO pipeline runs. + * @param codeCoverageConstantsBaseline - The code coverage constants required for fetching the baseline build artifacts. + * @param codeCoverageConstantsPR - The code coverage constants required for fetching the PR build artifacts. + * @param changedFiles - The list of files changed in the PR. + * @param logger - The logger to log messages. + */ +export async function getCodeCoverageReport( + adoToken: string, + codeCoverageConstantsBaseline: IAzureDevopsBuildCoverageConstants, + codeCoverageConstantsPR: IAzureDevopsBuildCoverageConstants, + changedFiles: string[], + logger?: CommandLogger, +): Promise { + const adoConnection = getAzureDevopsApi(adoToken, codeCoverageConstantsBaseline.orgUrl); + + const baselineBuildInfo = await getBaselineBuildMetrics( + codeCoverageConstantsBaseline, + adoConnection, + logger, + ).catch((error) => { + logger?.errorLog(`Error getting baseline build metrics: ${error}`); + throw error; + }); + + const adoConnectionForPR = getAzureDevopsApi(adoToken, codeCoverageConstantsPR.orgUrl); + + const prBuildInfo = await getBuildArtifactForSpecificBuild( + codeCoverageConstantsPR, + adoConnectionForPR, + logger, + ).catch((error) => { + logger?.errorLog(`Error getting PR build metrics: ${error}`); + throw error; + }); + + // Extract the coverage metrics for the baseline and PR builds. + const [coverageMetricsForBaseline, coverageMetricsForPr] = await Promise.all([ + getCoverageMetricsFromArtifact(baselineBuildInfo.artifactZip), + getCoverageMetricsFromArtifact(prBuildInfo.artifactZip), + ]); + + // Compare the code coverage metrics for the baseline and PR builds. + const comparisonData = compareCodeCoverage( + coverageMetricsForBaseline, + coverageMetricsForPr, + changedFiles, + ); + + return { + comparisonData, + baselineBuildMetrics: baselineBuildInfo, + }; +} diff --git a/build-tools/packages/build-cli/src/codeCoverage/compareCodeCoverage.ts b/build-tools/packages/build-cli/src/codeCoverage/compareCodeCoverage.ts new file mode 100644 index 000000000000..05d68b8e541d --- /dev/null +++ b/build-tools/packages/build-cli/src/codeCoverage/compareCodeCoverage.ts @@ -0,0 +1,213 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { CommandLogger } from "../logging.js"; +import type { CoverageMetric } from "./getCoverageMetrics.js"; + +// List of packages to be ignored from code coverage analysis. These are just prefixes. Reason is that when the package src code contains different +// folders, coverage report calculates coverage of sub folders separately. Also, for example we want to ignore all packages inside examples. So, checking +// prefix helps. If we want to ignore a specific package, we can add the package name directly. Also, the coverage report generates paths using dots as a +// separator for the path. +const codeCoverageComparisonIgnoreList: string[] = [ + "packages.common.core-interfaces", + "packages.common.core-utils", + "packages.common.driver-definitions", + "packages.common.container-definitions", + "packages.common.client-utils", + "packages.drivers.debugger", + "packages.drivers.file-driver", + "packages.drivers.odsp-driver-definitions", + "packages.drivers.replay-driver", + "packages.loader.test-loader-utils", + "packages.runtime.container-runtime-definitions", + "packages.runtime.datastore-definitions", + "packages.runtime.runtime-definitions", + "packages.runtime.test-runtime-utils", + "packages.test", + "packages.tools.changelog-generator-wrapper", + "packages.tools.devtools", + "packages.tools.fetch-tool", + "packages.tools.fluid-runner", + "packages.tools.replay-tool", +]; + +/** + * Type for the code coverage report generated by comparing the baseline and pr code coverage + */ +export interface CodeCoverageComparison { + /** + * Path of the package + */ + packagePath: string; + /** + * Line coverage in baseline build (as a percent) + */ + lineCoverageInBaseline: number; + /** + * Line coverage in pr build (as a percent) + */ + lineCoverageInPr: number; + /** + * difference between line coverage in pr build and baseline build (percentage points) + */ + lineCoverageDiff: number; + /** + * branch coverage in baseline build (as a percent) + */ + branchCoverageInBaseline: number; + /** + * branch coverage in pr build (as a percent) + */ + branchCoverageInPr: number; + /** + * difference between branch coverage in pr build and baseline build (percentage points) + */ + branchCoverageDiff: number; + /** + * Flag to indicate if the package is new + */ + isNewPackage: boolean; +} + +export interface CodeCoverageChangeForPackages { + codeCoverageComparisonForNewPackages: CodeCoverageComparison[]; + codeCoverageComparisonForExistingPackages: CodeCoverageComparison[]; +} + +/** + * Compares the code coverage for pr and baseline build and returns an array of objects with comparison results, + * one per package. + */ +export function compareCodeCoverage( + baselineCoverageReport: Map, + prCoverageReport: Map, + changedFiles: string[], +): CodeCoverageComparison[] { + const results: CodeCoverageComparison[] = []; + + const changedPackagesList = changedFiles.map((fileName) => { + const packagePath = fileName.split("/").slice(0, -1).join("."); + return packagePath; + }); + const changedPackages = new Set(changedPackagesList); + for (const changedPackage of changedPackages) { + let skip = false; + // Return if the package being updated in the PR is in the list of packages to be ignored. + // Also, ignore for now if the package is not in the packages folder. + for (const ignorePackageName of codeCoverageComparisonIgnoreList) { + if ( + changedPackage.startsWith(ignorePackageName) || + !changedPackage.startsWith("packages.") + ) { + skip = true; + break; + } + } + + if (skip) { + continue; + } + + const prCoverageMetrics = prCoverageReport.get(changedPackage); + const baselineCoverageMetrics = baselineCoverageReport.get(changedPackage); + const isNewPackage = baselineCoverageMetrics === undefined; + if (prCoverageMetrics === undefined) { + continue; + } + + let lineCoverageInBaseline = 0; + let branchCoverageInBaseline = 0; + const lineCoverageInPr = prCoverageMetrics.lineCoverage; + const branchCoverageInPr = prCoverageMetrics.branchCoverage; + + if (baselineCoverageMetrics) { + lineCoverageInBaseline = baselineCoverageMetrics.lineCoverage; + branchCoverageInBaseline = baselineCoverageMetrics.branchCoverage; + } + + results.push({ + packagePath: changedPackage, + lineCoverageInBaseline, + lineCoverageInPr, + lineCoverageDiff: lineCoverageInPr - lineCoverageInBaseline, + branchCoverageInBaseline, + branchCoverageInPr, + branchCoverageDiff: branchCoverageInPr - branchCoverageInBaseline, + isNewPackage, + }); + } + + return results; +} + +/** + * Method that returns list of packages with code coverage changes. + * @param codeCoverageComparisonData - The comparison data between baseline and pr test coverage + * @param logger - The logger to log messages. + */ +export function getPackagesWithCodeCoverageChanges( + codeCoverageComparisonData: CodeCoverageComparison[], + logger?: CommandLogger, +): CodeCoverageChangeForPackages { + // Find new packages that do not have test setup and are being impacted by changes in the PR + const newPackagesIdentifiedByCodeCoverage = codeCoverageComparisonData.filter( + (codeCoverageReport) => codeCoverageReport.isNewPackage, + ); + logger?.verbose(`Found ${newPackagesIdentifiedByCodeCoverage.length} new packages`); + + // Find existing packages that have reported a change in coverage for the current PR + const existingPackagesWithCoverageChange = codeCoverageComparisonData.filter( + (codeCoverageReport) => + codeCoverageReport.branchCoverageDiff !== 0 || codeCoverageReport.lineCoverageDiff !== 0, + ); + logger?.verbose( + `Found ${existingPackagesWithCoverageChange.length} packages with code coverage changes`, + ); + + return { + codeCoverageComparisonForNewPackages: newPackagesIdentifiedByCodeCoverage, + codeCoverageComparisonForExistingPackages: existingPackagesWithCoverageChange, + }; +} + +/** + * Method that returns whether the code coverage comparison check passed or not. + * @param codeCoverageChangeForPackages - The comparison data for packages with code coverage changes. + * @param logger - The logger to log messages. + */ +export function isCodeCoverageCriteriaPassed( + codeCoverageChangeForPackages: CodeCoverageChangeForPackages, + logger?: CommandLogger, +): boolean { + const { codeCoverageComparisonForNewPackages, codeCoverageComparisonForExistingPackages } = + codeCoverageChangeForPackages; + const packagesWithNotableRegressions = codeCoverageComparisonForExistingPackages.filter( + (codeCoverageReport: CodeCoverageComparison) => + codeCoverageReport.branchCoverageDiff < -1 || codeCoverageReport.lineCoverageDiff < -1, + ); + + logger?.verbose( + `Found ${packagesWithNotableRegressions.length} existing packages with notable regressions`, + ); + + // Code coverage for the newly added package should be less than 50% to fail. + const newPackagesWithNotableRegressions = codeCoverageComparisonForNewPackages.filter( + (codeCoverageReport) => + codeCoverageReport.branchCoverageInPr < 50 || codeCoverageReport.lineCoverageInPr < 50, + ); + + logger?.verbose( + `Found ${newPackagesWithNotableRegressions.length} new packages with notable regressions`, + ); + let success: boolean = false; + if ( + newPackagesWithNotableRegressions.length === 0 && + packagesWithNotableRegressions.length === 0 + ) { + success = true; + } + + return success; +} diff --git a/build-tools/packages/build-cli/src/codeCoverage/getCommentForCodeCoverage.ts b/build-tools/packages/build-cli/src/codeCoverage/getCommentForCodeCoverage.ts new file mode 100644 index 000000000000..9fc922909cba --- /dev/null +++ b/build-tools/packages/build-cli/src/codeCoverage/getCommentForCodeCoverage.ts @@ -0,0 +1,128 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { IBuildMetrics } from "../library/azureDevops/getBaselineBuildMetrics.js"; +import type { + CodeCoverageChangeForPackages, + CodeCoverageComparison, +} from "./compareCodeCoverage.js"; + +const codeCoverageDetailsHeader = ``; + +/** + * Method that returns the comment to be posted on PRs about code coverage + * @param packagesListWithCodeCoverageChanges - The comparison data for packages with code coverage changes. + * @param baselineBuildInfo - The baseline build information. + * @param success - Flag to indicate if the code coverage comparison check passed or not + * @returns Comment to be posted on the PR, and whether the code coverage comparison check passed or not + */ +export function getCommentForCodeCoverageDiff( + packagesListWithCodeCoverageChanges: CodeCoverageChangeForPackages, + baselineBuildInfo: IBuildMetrics, + success: boolean, +): string { + const { codeCoverageComparisonForNewPackages, codeCoverageComparisonForExistingPackages } = + packagesListWithCodeCoverageChanges; + + let coverageSummaryForImpactedPackages = ""; + let coverageSummaryForNewPackages = ""; + + if ( + codeCoverageComparisonForExistingPackages.length === 0 && + codeCoverageComparisonForNewPackages.length === 0 + ) { + coverageSummaryForImpactedPackages = `No packages impacted by the change.`; + } + + if (codeCoverageComparisonForExistingPackages.length > 0) { + coverageSummaryForImpactedPackages = getCodeCoverageSummary( + codeCoverageComparisonForExistingPackages, + ); + } + + if (codeCoverageComparisonForNewPackages.length > 0) { + coverageSummaryForNewPackages = getCodeCoverageSummary( + codeCoverageComparisonForNewPackages, + ); + } + return [ + "## Code Coverage Summary", + coverageSummaryForImpactedPackages, + coverageSummaryForNewPackages, + getSummaryFooter(baselineBuildInfo), + success + ? "### Code coverage comparison check passed!!" + : "### Code coverage comparison check failed!!", + ].join("\n\n"); +} + +const getSummaryFooter = (baselineBuildInfo: IBuildMetrics): string => { + return `

Baseline commit: ${ + baselineBuildInfo.build.sourceVersion + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + }
Baseline build: ${ + baselineBuildInfo.build.id + }
Happy Coding!!

`; +}; + +const getCodeCoverageSummary = ( + codeCoverageComparisonReport: CodeCoverageComparison[], +): string => { + const summary = codeCoverageComparisonReport + .sort((report1, report2) => report1.branchCoverageDiff - report2.branchCoverageDiff) + .map((coverageReport) => getCodeCoverageSummaryForPackages(coverageReport)) + .reduce((prev, current) => prev + current); + + return summary; +}; + +const getCodeCoverageSummaryForPackages = (coverageReport: CodeCoverageComparison): string => { + const metrics = codeCoverageDetailsHeader + getMetricRows(coverageReport); + + return `
${getColorGlyph(coverageReport.branchCoverageDiff)} ${ + coverageReport.packagePath + }: ${formatDiff(coverageReport.branchCoverageDiff)}${metrics}
Metric NameBaseline coveragePR coverageCoverage Diff
`; +}; + +const getColorGlyph = (codeCoverageBranchDiff: number): string => { + if (codeCoverageBranchDiff === 0) { + return '■'; + } + + if (codeCoverageBranchDiff > 0) { + return '⯅'; + } + + return '⯆'; +}; + +const formatDiff = (coverageDiff: number): string => { + if (coverageDiff === 0) { + return "No change"; + } + return `${coverageDiff.toFixed(2)}%`; +}; + +const getMetricRows = (codeCoverageComparisonReport: CodeCoverageComparison): string => { + const glyphForLineCoverage = getColorGlyph(codeCoverageComparisonReport.lineCoverageDiff); + const glyphForBranchCoverage = getColorGlyph( + codeCoverageComparisonReport.branchCoverageDiff, + ); + + return ( + ` + Branch Coverage + ${formatDiff(codeCoverageComparisonReport.branchCoverageInBaseline)} + ${formatDiff(codeCoverageComparisonReport.branchCoverageInPr)} + ${glyphForBranchCoverage} ${formatDiff(codeCoverageComparisonReport.branchCoverageDiff)} + ` + + ` + Line Coverage + ${formatDiff(codeCoverageComparisonReport.lineCoverageInBaseline)} + ${formatDiff(codeCoverageComparisonReport.lineCoverageInPr)} + ${glyphForLineCoverage} ${formatDiff(codeCoverageComparisonReport.lineCoverageDiff)} + ` + ); +}; diff --git a/build-tools/packages/build-cli/src/codeCoverage/getCoverageMetrics.ts b/build-tools/packages/build-cli/src/codeCoverage/getCoverageMetrics.ts new file mode 100644 index 000000000000..6407833f72bf --- /dev/null +++ b/build-tools/packages/build-cli/src/codeCoverage/getCoverageMetrics.ts @@ -0,0 +1,114 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import JSZip from "jszip"; +import { Parser } from "xml2js"; +import type { CommandLogger } from "../logging.js"; + +/** + * The type for the coverage report, containing the line coverage and branch coverage(in percentage) + */ +export interface CoverageMetric { + lineCoverage: number; + branchCoverage: number; +} + +interface XmlCoverageReportSchema { + coverage: { + packages: XmlCoverageReportSchemaForPackage[]; + }; +} + +interface XmlCoverageReportSchemaForPackage { + package: [ + { + "$": { + name: string; + "line-rate": string; + "branch-rate": string; + }; + }, + ]; +} + +const extractCoverageMetrics = ( + xmlForCoverageReportFromArtifact: XmlCoverageReportSchema, +): Map => { + const report: Map = new Map(); + const coverageForPackagesResult = + xmlForCoverageReportFromArtifact.coverage.packages[0]?.package; + + for (const coverageForPackage of coverageForPackagesResult) { + const packagePath = coverageForPackage.$.name; + const lineCoverage = Number.parseFloat(coverageForPackage.$["line-rate"]) * 100; + const branchCoverage = Number.parseFloat(coverageForPackage.$["branch-rate"]) * 100; + if (packagePath && !Number.isNaN(lineCoverage) && !Number.isNaN(branchCoverage)) { + report.set(packagePath, { + lineCoverage, + branchCoverage, + }); + } + } + return report; +}; + +/** + * Method that returns the coverage report for the build from the artifact. + * @param baselineZip - zipped coverage files for the build + * @param logger - The logger to log messages. + * @returns an map of coverage metrics for build containing packageName, lineCoverage and branchCoverage + */ +export const getCoverageMetricsFromArtifact = async ( + artifactZip: JSZip, + logger?: CommandLogger, +): Promise> => { + const coverageReportsFiles: string[] = []; + // eslint-disable-next-line unicorn/no-array-for-each -- required as JSZip does not implement [Symbol.iterator]() which is required by for...of + artifactZip.forEach((filePath) => { + if (filePath.endsWith("cobertura-coverage-patched.xml")) + coverageReportsFiles.push(filePath); + }); + + let coverageMetricsForBaseline: Map = new Map(); + const xmlParser = new Parser(); + + try { + logger?.info(`${coverageReportsFiles.length} coverage data files found.`); + + for (const coverageReportFile of coverageReportsFiles) { + const jsZipObject = artifactZip.file(coverageReportFile); + if (jsZipObject === undefined) { + logger?.warning( + `could not find file ${coverageReportFile} in the code coverage artifact`, + ); + } + + // eslint-disable-next-line no-await-in-loop -- Since we only need 1 report file, it is easier to run it serially rather than extracting all jsZipObjects and then awaiting promises in parallel + const coverageReportXML = await jsZipObject?.async("nodebuffer"); + if (coverageReportXML !== undefined) { + xmlParser.parseString( + coverageReportXML, + (err: Error | null, result: unknown): void => { + if (err) { + console.warn(`Error processing file ${coverageReportFile}: ${err}`); + return; + } + coverageMetricsForBaseline = extractCoverageMetrics( + result as XmlCoverageReportSchema, + ); + }, + ); + } + if (coverageMetricsForBaseline.size > 0) { + break; + } + } + } catch (error) { + logger?.warning(`Error encountered with reading files: ${error}`); + } + + logger?.info(`${coverageMetricsForBaseline.size} packages with coverage data found.`); + return coverageMetricsForBaseline; +}; diff --git a/build-tools/packages/build-cli/src/commands/report/codeCoverage.ts b/build-tools/packages/build-cli/src/commands/report/codeCoverage.ts new file mode 100644 index 000000000000..9d9494be85b2 --- /dev/null +++ b/build-tools/packages/build-cli/src/commands/report/codeCoverage.ts @@ -0,0 +1,165 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { Flags } from "@oclif/core"; +import { getCodeCoverageReport } from "../../codeCoverage/codeCoveragePr.js"; +import { + getPackagesWithCodeCoverageChanges, + isCodeCoverageCriteriaPassed, +} from "../../codeCoverage/compareCodeCoverage.js"; +import { getCommentForCodeCoverageDiff } from "../../codeCoverage/getCommentForCodeCoverage.js"; +import { type IAzureDevopsBuildCoverageConstants } from "../../library/azureDevops/constants.js"; +import { + type GitHubProps, + createOrUpdateCommentOnPr, + getChangedFilePaths, +} from "../../library/githubRest.js"; +import { BaseCommand } from "../../library/index.js"; + +// Unique identifier for the comment made on the PR. This is used to identify the comment +// and update it based on a new build. +const commentIdentifier = ``; + +export default class ReportCodeCoverageCommand extends BaseCommand< + typeof ReportCodeCoverageCommand +> { + static readonly description = "Run comparison of code coverage stats"; + + static readonly flags = { + adoBuildId: Flags.integer({ + description: "Azure DevOps build ID.", + env: "ADO_BUILD_ID", + required: true, + }), + adoApiToken: Flags.string({ + description: "Token to get auth for accessing ADO builds.", + env: "ADO_API_TOKEN", + required: true, + }), + githubApiToken: Flags.string({ + description: "Token to get auth for accessing Github PR.", + env: "GITHUB_API_TOKEN", + required: true, + }), + adoCIBuildDefinitionIdBaseline: Flags.integer({ + description: "Build definition/pipeline number/id for the baseline build.", + env: "ADO_CI_BUILD_DEFINITION_ID_BASELINE", + required: true, + }), + adoCIBuildDefinitionIdPR: Flags.integer({ + description: "Build definition/pipeline number/id for the PR build.", + env: "ADO_CI_BUILD_DEFINITION_ID_PR", + required: true, + }), + codeCoverageAnalysisArtifactNameBaseline: Flags.string({ + description: "Code coverage artifact name for the baseline build.", + env: "CODE_COVERAGE_ANALYSIS_ARTIFACT_NAME_BASELINE", + required: true, + }), + codeCoverageAnalysisArtifactNamePR: Flags.string({ + description: "Code coverage artifact name for the PR build.", + env: "CODE_COVERAGE_ANALYSIS_ARTIFACT_NAME_PR", + required: true, + }), + githubPRNumber: Flags.integer({ + description: "Github PR number.", + env: "GITHUB_PR_NUMBER", + required: true, + }), + githubRepositoryName: Flags.string({ + description: "Github repository name.", + env: "GITHUB_REPOSITORY_NAME", + required: true, + }), + githubRepositoryOwner: Flags.string({ + description: "Github repository owner.", + env: "GITHUB_REPOSITORY_OWNER", + required: true, + }), + targetBranchName: Flags.string({ + description: "Target branch name.", + env: "TARGET_BRANCH_NAME", + required: true, + }), + ...BaseCommand.flags, + }; + + public async run(): Promise { + const { flags } = this; + + const codeCoverageConstantsForBaseline: IAzureDevopsBuildCoverageConstants = { + orgUrl: "https://dev.azure.com/fluidframework", + projectName: "public", + ciBuildDefinitionId: flags.adoCIBuildDefinitionIdBaseline, + artifactName: flags.codeCoverageAnalysisArtifactNameBaseline, + branch: flags.targetBranchName, + buildsToSearch: 50, + }; + + const codeCoverageConstantsForPR: IAzureDevopsBuildCoverageConstants = { + orgUrl: "https://dev.azure.com/fluidframework", + projectName: "public", + ciBuildDefinitionId: flags.adoCIBuildDefinitionIdPR, + artifactName: flags.codeCoverageAnalysisArtifactNamePR, + buildsToSearch: 20, + buildId: flags.adoBuildId, + }; + + const githubProps: GitHubProps = { + owner: flags.githubRepositoryOwner, + repo: flags.githubRepositoryName, + token: flags.githubApiToken, + }; + + // Get the paths of the files that have changed in the PR relative to root of the repo. + // This is used to determine which packages have been affect so that we can do code coverage + // analysis on those packages only. + const changedFiles = await getChangedFilePaths(githubProps, flags.githubPRNumber); + + let commentMessage: string = ""; + const report = await getCodeCoverageReport( + flags.adoApiToken, + codeCoverageConstantsForBaseline, + codeCoverageConstantsForPR, + changedFiles, + this.logger, + ).catch((error: Error) => { + commentMessage = "## Code Coverage Summary\n\nError getting code coverage report"; + this.logger.errorLog(`Error getting code coverage report: ${error}`); + return undefined; + }); + + // Don't fail if we can not compare the code coverage due to an error. + let success: boolean = true; + if (report !== undefined) { + const packagesListWithCodeCoverageChanges = getPackagesWithCodeCoverageChanges( + report.comparisonData, + this.logger, + ); + + success = isCodeCoverageCriteriaPassed(packagesListWithCodeCoverageChanges, this.logger); + + commentMessage = getCommentForCodeCoverageDiff( + packagesListWithCodeCoverageChanges, + report.baselineBuildMetrics, + success, + ); + } + + const messageContentWithIdentifier = `${commentIdentifier}\n\n${commentMessage}`; + + await createOrUpdateCommentOnPr( + githubProps, + flags.githubPRNumber, + messageContentWithIdentifier, + commentIdentifier, + ); + + // Fail the build if the code coverage analysis shows that a regression has been found. + if (!success) { + this.error("Code coverage failed", { exit: 255 }); + } + } +} diff --git a/build-tools/packages/build-cli/src/library/azureDevops/constants.ts b/build-tools/packages/build-cli/src/library/azureDevops/constants.ts new file mode 100644 index 000000000000..31b3be592b7c --- /dev/null +++ b/build-tools/packages/build-cli/src/library/azureDevops/constants.ts @@ -0,0 +1,43 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +export interface IAzureDevopsBuildCoverageConstants { + /** + * URL for the ADO org. + */ + orgUrl: string; + + /** + * The ADO project which contains the pipeline that generates the code coverage report artifacts. + */ + projectName: string; + + /** + * The ADO ID of the pipeline (aka `definitionId`) that runs against main when PRs are merged and + * generates the code coverage artifacts. + */ + ciBuildDefinitionId: number; + + /** + * The name of the build artifact that contains the artifact to be used as for analysis. + */ + artifactName: string; + + /** + * The number of most recent ADO builds to pull when searching for a particular build. Pulling more + * builds takes longer, but may be useful when there are a high volume of commits/builds. + */ + buildsToSearch?: number; + + /** + * The branch for which the build is searched. + */ + branch?: string; + + /** + * Current Build ID of the PR for which code coverage analysis will be done. + */ + buildId?: number; +} diff --git a/build-tools/packages/build-cli/src/library/azureDevops/getBaselineBuildMetrics.ts b/build-tools/packages/build-cli/src/library/azureDevops/getBaselineBuildMetrics.ts new file mode 100644 index 000000000000..151b3ad52dff --- /dev/null +++ b/build-tools/packages/build-cli/src/library/azureDevops/getBaselineBuildMetrics.ts @@ -0,0 +1,189 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import assert from "node:assert"; +import { getZipObjectFromArtifact } from "@fluidframework/bundle-size-tools"; +import type { WebApi } from "azure-devops-node-api"; +import { BuildResult } from "azure-devops-node-api/interfaces/BuildInterfaces.js"; +import type { Build } from "azure-devops-node-api/interfaces/BuildInterfaces.js"; +import type JSZip from "jszip"; +import type { CommandLogger } from "../../logging.js"; +import type { IAzureDevopsBuildCoverageConstants } from "./constants.js"; +import { getBuild, getBuilds } from "./utils.js"; + +export interface IBuildMetrics { + build: Build & { id: number }; + /** + * The artifact that was published by the PR build in zip format + */ + artifactZip: JSZip; +} + +/** + * Method that returns the build artifact for a baseline build. + * @param azureDevopsBuildCoverageConstants - Code coverage constants for the project. + * @param adoConnection - The connection to the Azure DevOps API + * @param logger - The logger to log messages. + */ +export async function getBaselineBuildMetrics( + azureDevopsBuildCoverageConstants: IAzureDevopsBuildCoverageConstants, + adoConnection: WebApi, + logger?: CommandLogger, +): Promise { + const recentBuilds = await getBuilds(adoConnection, { + project: azureDevopsBuildCoverageConstants.projectName, + definitions: [azureDevopsBuildCoverageConstants.ciBuildDefinitionId], + branch: + azureDevopsBuildCoverageConstants.branch === undefined + ? undefined + : `refs/heads/${azureDevopsBuildCoverageConstants.branch}`, + maxBuildsPerDefinition: azureDevopsBuildCoverageConstants.buildsToSearch ?? 50, + }); + + let baselineBuild: Build | undefined; + let baselineArtifactZip: JSZip | undefined; + for (const build of recentBuilds) { + if (build.result !== BuildResult.Succeeded) { + continue; + } + + // Baseline build does not have id + if (build.id === undefined) { + const message = `Baseline build does not have a build id`; + logger?.warning(message); + throw new Error(message); + } + + // Baseline build succeeded + logger?.verbose(`Found baseline build with id: ${build.id}`); + logger?.verbose(`projectName: ${azureDevopsBuildCoverageConstants.projectName}`); + logger?.verbose( + `codeCoverageAnalysisArtifactName: ${azureDevopsBuildCoverageConstants.artifactName}`, + ); + + // eslint-disable-next-line no-await-in-loop + baselineArtifactZip = await getZipObjectFromArtifact( + adoConnection, + azureDevopsBuildCoverageConstants.projectName, + build.id, + `${azureDevopsBuildCoverageConstants.artifactName}_${build.id}`, + ).catch((error: Error) => { + logger?.warning( + `Failed to fetch and/or unzip artifact '${azureDevopsBuildCoverageConstants.artifactName}' from CI build. Cannot generate analysis at this time`, + ); + logger?.warning(`Error: ${error.message}`); + logger?.warning(`Error stack: ${error.stack}`); + return undefined; + }); + + // For reasons that I don't understand, the "undefined" string is omitted in the log output, which makes the + // output very confusing. The string is capitalized here and elsewhere in this file as a workaround. + logger?.verbose(`Baseline Zip === UNDEFINED: ${baselineArtifactZip === undefined}`); + + // Successful baseline build does not have the needed build artifacts + if (baselineArtifactZip === undefined) { + logger?.warning( + `Trying backup builds when successful baseline build does not have the needed build artifacts ${build.id}`, + ); + continue; + } + // Found usable baseline zip, so break out of the loop early. + baselineBuild = build; + break; + } + + // Unable to find a usable baseline + if (baselineArtifactZip === undefined) { + const message = `Could not find a usable baseline build`; + logger?.warning(message); + throw new Error(message); + } + + if (!baselineBuild) { + const message = `Could not find baseline build for CI`; + logger?.warning(message); + throw new Error(message); + } + + // Baseline build does not have id + if (baselineBuild.id === undefined) { + const message = `Baseline build does not have a build id`; + logger?.warning(message); + throw new Error(message); + } + + logger?.verbose(`Found baseline build with id: ${baselineBuild.id}`); + + return { + build: { ...baselineBuild, id: baselineBuild.id }, + artifactZip: baselineArtifactZip, + }; +} + +/** + * Method that returns the build artifact for a specific build. + * @param azureDevopsBuildCoverageConstants - Code coverage constants for the project. + * @param adoConnection - The connection to the Azure DevOps API + * @param logger - The logger to log messages. + */ +export async function getBuildArtifactForSpecificBuild( + azureDevopsBuildCoverageConstants: IAzureDevopsBuildCoverageConstants, + adoConnection: WebApi, + logger?: CommandLogger, +): Promise { + assert(azureDevopsBuildCoverageConstants.buildId !== undefined, "buildId is required"); + logger?.verbose(`The buildId id ${azureDevopsBuildCoverageConstants.buildId}`); + + const build: Build = await getBuild( + adoConnection, + { + project: azureDevopsBuildCoverageConstants.projectName, + definitions: [azureDevopsBuildCoverageConstants.ciBuildDefinitionId], + maxBuildsPerDefinition: azureDevopsBuildCoverageConstants.buildsToSearch ?? 20, + }, + azureDevopsBuildCoverageConstants.buildId, + ); + + // Build does not have id + if (build.id === undefined) { + const message = `build does not have a build id`; + logger?.warning(message); + throw new Error(message); + } + + logger?.verbose(`Found build with id: ${build.id}`); + logger?.verbose(`projectName: ${azureDevopsBuildCoverageConstants.projectName}`); + logger?.verbose( + `codeCoverageAnalysisArtifactName: ${azureDevopsBuildCoverageConstants.artifactName}`, + ); + + const artifactZip: JSZip | undefined = await getZipObjectFromArtifact( + adoConnection, + azureDevopsBuildCoverageConstants.projectName, + build.id, + `${azureDevopsBuildCoverageConstants.artifactName}_${build.id}`, + ).catch((error: Error) => { + logger?.warning( + `Failed to fetch and/or unzip artifact '${azureDevopsBuildCoverageConstants.artifactName}' from CI build. Cannot generate analysis at this time`, + ); + logger?.warning(`Error: ${error.message}`); + logger?.warning(`Error stack: ${error.stack}`); + return undefined; + }); + + // For reasons that I don't understand, the "undefined" string is omitted in the log output, which makes the + // output very confusing. The string is capitalized here and elsewhere in this file as a workaround. + logger?.verbose(`Artifact Zip === UNDEFINED: ${artifactZip === undefined}`); + if (artifactZip === undefined) { + const message = `Could not find a usable artifact`; + logger?.warning(message); + throw new Error(message); + } + + return { + build: { ...build, id: build.id }, + artifactZip, + }; +} diff --git a/build-tools/packages/build-cli/src/library/azureDevops/utils.ts b/build-tools/packages/build-cli/src/library/azureDevops/utils.ts new file mode 100644 index 000000000000..26ad8629b011 --- /dev/null +++ b/build-tools/packages/build-cli/src/library/azureDevops/utils.ts @@ -0,0 +1,82 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { WebApi } from "azure-devops-node-api"; +import { + type Build, + BuildQueryOrder, +} from "azure-devops-node-api/interfaces/BuildInterfaces.js"; + +export interface GetBuildOptions { + /** + * The ADO project name + */ + project: string; + + /** + * An array of ADO definitions that should be considered for this query. + */ + definitions: number[]; + + /** + * An optional set of tags that should be on the returned builds. + */ + tagFilters?: string[]; + + /** + * An upper limit on the number of queries to return. Can be used to improve performance + */ + maxBuildsPerDefinition?: number; + + /** + * Name of the branch for which the builds are being fetched. + */ + branch?: string; +} + +/** + * A wrapper around the terrible API signature for ADO getBuilds + */ +export async function getBuilds( + adoConnection: WebApi, + options: GetBuildOptions, + build_id?: string, +): Promise { + const buildApi = await adoConnection.getBuildApi(); + + return buildApi.getBuilds( + options.project, + options.definitions, + undefined /* queues */, + build_id, + undefined /* minTime */, + undefined /* maxTime */, + undefined /* requestedFor */, + undefined /* reasonFilter */, + undefined /* BuildStatus */, + undefined /* BuildResult */, + options.tagFilters, + undefined /* properties */, + undefined /* top */, + undefined /* continuationToken */, + options.maxBuildsPerDefinition, + undefined /* deletedFilter */, + BuildQueryOrder.QueueTimeDescending, + options.branch, + ); +} + +/** + * A wrapper around the API signature for ADO getBuild + */ +export async function getBuild( + adoConnection: WebApi, + options: GetBuildOptions, + buildId: number, +): Promise { + const buildApi = await adoConnection.getBuildApi(); + + return buildApi.getBuild(options.project, buildId); +} diff --git a/build-tools/packages/build-cli/src/library/githubRest.ts b/build-tools/packages/build-cli/src/library/githubRest.ts index b0f88d988dce..a38ce66623a3 100644 --- a/build-tools/packages/build-cli/src/library/githubRest.ts +++ b/build-tools/packages/build-cli/src/library/githubRest.ts @@ -110,3 +110,80 @@ export async function isPrApprovedByUsers( const approved = reviewers.some((user) => approvers.has(user)); return approved; } + +/** + * Creates or modifies a single review comment on a PR. The comment is identified with a unique identifier, so the same comment is updated on repeated calls. + * + * @param github - Details about the GitHub repo and auth to use. + * @param prNumber - Pull request number. + * @param body - review comment body to be posted. + * @param commentIdentifier - unique identifier for the comment to be updated. + * + * @returns id of the comment that was updated. + */ +export async function createOrUpdateCommentOnPr( + { owner, repo, token }: GitHubProps, + prNumber: number, + body: string, + commentIdentifier: string, +): Promise { + const octokit = new Octokit({ auth: token }); + + // List of review comments for the pull request + const { data: comments } = await octokit.pulls.listReviews({ + owner, + repo, + pull_number: prNumber, + }); + + let commentId: number | undefined; + // Log the comments to find the comment_id + for (const comment of comments) { + if (comment.body.startsWith(commentIdentifier)) { + commentId = comment.id; + break; + } + } + + if (commentId === undefined) { + const response = await octokit.pulls.createReview({ + owner, + repo, + pull_number: prNumber, + event: "COMMENT", + body, + }); + return response.data.id; + } + // Update PR review comment + const { data } = await octokit.pulls.updateReview({ + owner, + repo, + pull_number: prNumber, + body, + review_id: commentId, + }); + return data.id; +} + +/** + * Api to get the changed file paths in a PR. The paths are relative to the root of the repo. + * @param github - Details about the GitHub repo and auth to use. + * @param prNumber - Pr number for which the changed files paths are to be fetched + * @returns - List of file paths that are changed in the PR + */ +export async function getChangedFilePaths( + { owner, repo, token }: GitHubProps, + prNumber: number, +): Promise { + const octokit = new Octokit({ auth: token }); + + // List of files changed in the pull request + const { data: files } = await octokit.pulls.listFiles({ + owner, + repo, + pull_number: prNumber, + }); + const fileNames = files.map((file) => file.filename); + return fileNames; +} diff --git a/build-tools/pnpm-lock.yaml b/build-tools/pnpm-lock.yaml index 76216a53377b..dad0b629b90f 100644 --- a/build-tools/pnpm-lock.yaml +++ b/build-tools/pnpm-lock.yaml @@ -135,6 +135,9 @@ importers: async: specifier: ^3.2.4 version: 3.2.4 + azure-devops-node-api: + specifier: ^11.2.0 + version: 11.2.0 chalk: specifier: ^5.3.0 version: 5.3.0 @@ -186,6 +189,9 @@ importers: jssm: specifier: 5.98.2 version: 5.98.2(patch_hash=jvxmn6yyt5s6cxokjpzzkp2gqy) + jszip: + specifier: ^3.10.1 + version: 3.10.1 latest-version: specifier: ^5.1.0 version: 5.1.0 @@ -270,6 +276,9 @@ importers: unist-util-visit: specifier: ^5.0.0 version: 5.0.0 + xml2js: + specifier: ^0.5.0 + version: 0.5.0 devDependencies: '@biomejs/biome': specifier: ~1.8.3 @@ -334,6 +343,9 @@ importers: '@types/unist': specifier: ^3.0.3 version: 3.0.3 + '@types/xml2js': + specifier: ^0.4.11 + version: 0.4.11 c8: specifier: ^7.14.0 version: 7.14.0 @@ -1880,7 +1892,7 @@ packages: clean-stack: 3.0.1 cli-progress: 3.12.0 color: 4.2.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.5(supports-color@8.1.1) ejs: 3.1.10 get-package-type: 0.1.0 globby: 11.1.0 @@ -1888,7 +1900,7 @@ packages: indent-string: 4.0.0 is-wsl: 2.2.0 js-yaml: 3.14.1 - minimatch: 9.0.4 + minimatch: 9.0.5 natural-orderby: 2.0.3 object-treeify: 1.1.33 password-prompt: 1.1.3 @@ -2571,7 +2583,7 @@ packages: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==} dependencies: '@types/minimatch': 3.0.5 - '@types/node': 18.18.6 + '@types/node': 18.18.7 /@types/http-cache-semantics@4.0.4: resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} @@ -2749,6 +2761,12 @@ packages: /@types/wrap-ansi@3.0.0: resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} + /@types/xml2js@0.4.11: + resolution: {integrity: sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==} + dependencies: + '@types/node': 18.18.7 + dev: true + /@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.57.0)(typescript@5.4.5): resolution: {integrity: sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA==} engines: {node: ^16.0.0 || >=18.0.0} @@ -6283,6 +6301,7 @@ packages: /glob@7.2.0: resolution: {integrity: sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -6294,6 +6313,7 @@ packages: /glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -6305,6 +6325,7 @@ packages: /glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -8241,13 +8262,6 @@ packages: dependencies: brace-expansion: 2.0.1 - /minimatch@9.0.4: - resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==} - engines: {node: '>=16 || 14 >=14.17'} - dependencies: - brace-expansion: 2.0.1 - dev: true - /minimatch@9.0.5: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} @@ -9850,6 +9864,10 @@ packages: /safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + /sax@1.4.1: + resolution: {integrity: sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==} + dev: false + /schema-utils@3.2.0: resolution: {integrity: sha512-0zTyLGyDJYd/MBxG1AhJkKa6fpEBds4OQO2ut0w7OYG+ZGhGea09lijvzsqegYSik88zc7cUtIlnnO+/BvD6gQ==} engines: {node: '>= 10.13.0'} @@ -11335,6 +11353,19 @@ packages: resolution: {integrity: sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==} engines: {node: '>=12'} + /xml2js@0.5.0: + resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==} + engines: {node: '>=4.0.0'} + dependencies: + sax: 1.4.1 + xmlbuilder: 11.0.1 + dev: false + + /xmlbuilder@11.0.1: + resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} + engines: {node: '>=4.0'} + dev: false + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'}