From 2f778c6fa2cb6add6c463afac6e3e242e5d93a40 Mon Sep 17 00:00:00 2001 From: Free Wortley Date: Wed, 15 Feb 2023 17:28:48 -0800 Subject: [PATCH 1/4] MVP of CLI to file PRs with Package Updates There are still a few bugs left to shake out here, but the code is 99% of the way there now. Example PR generated with this command: `yarn run dev github-pr replace-package freeqaz/jira_clone --githubToken --old js-yaml@^3.13.1 --new js-yaml@^3.14.0` https://github.com/freeqaz/jira_clone/pull/2 Bugs left: - [ ] Figure out why packages are marked "extraneous" in the generated lockfile - [ ] Name the folder where these packages are inserted to be the same as the repo (the package-lock gets a new name currently and it's annoying) Items left: - [ ] Call this module from the backend by adding a new Endpoint for it - [ ] Write the front-end changes to call the endpoint - [ ] Write some basic unit tests to test this functionality --- .idea/dbnavigator.xml | 1 + .../down.sql | 4 +- .../up.sql | 4 +- lunatrace/npm-package-cli/package.json | 9 +- .../src/commands/github-pr/replace-package.ts | 107 +++++++++ .../src/commands/replace-package/index.ts | 55 ++--- .../src/commands/show-tree/index.ts | 2 +- .../src/package/github-pr/index.ts | 225 ++++++++++++++++++ .../src/package/replace-package/index.ts | 57 +++++ .../{ => replace-package}/package-tree.ts | 18 +- yarn.lock | 142 +++++++++++ 11 files changed, 565 insertions(+), 59 deletions(-) create mode 100644 lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts create mode 100644 lunatrace/npm-package-cli/src/package/github-pr/index.ts create mode 100644 lunatrace/npm-package-cli/src/package/replace-package/index.ts rename lunatrace/npm-package-cli/src/package/{ => replace-package}/package-tree.ts (89%) diff --git a/.idea/dbnavigator.xml b/.idea/dbnavigator.xml index 54a5176eb..400de5344 100644 --- a/.idea/dbnavigator.xml +++ b/.idea/dbnavigator.xml @@ -49,6 +49,7 @@ + diff --git a/lunatrace/bsl/hasura/migrations/lunatrace/1672788403469_add_cisa_known_exploited_vulnerabilities/down.sql b/lunatrace/bsl/hasura/migrations/lunatrace/1672788403469_add_cisa_known_exploited_vulnerabilities/down.sql index 5ce32bb12..aa367b0f6 100644 --- a/lunatrace/bsl/hasura/migrations/lunatrace/1672788403469_add_cisa_known_exploited_vulnerabilities/down.sql +++ b/lunatrace/bsl/hasura/migrations/lunatrace/1672788403469_add_cisa_known_exploited_vulnerabilities/down.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS vulnerability.cisa_known_exploited CASCADE; DROP INDEX IF EXISTS vulnerability_equivalent_b_idx; -DROP FUNCTION vulnerability.cisa_known_exploited_vulnerability; -DROP FUNCTION vulnerability.vulnerability_cisa_known_exploited; +DROP FUNCTION IF EXISTS vulnerability.cisa_known_exploited_vulnerability; +DROP FUNCTION IF EXISTS vulnerability.vulnerability_cisa_known_exploited; diff --git a/lunatrace/bsl/hasura/migrations/lunatrace/1674685930023_add_cve_id_to_all_vulns/up.sql b/lunatrace/bsl/hasura/migrations/lunatrace/1674685930023_add_cve_id_to_all_vulns/up.sql index 6118ba7cf..fbfc6cc67 100644 --- a/lunatrace/bsl/hasura/migrations/lunatrace/1674685930023_add_cve_id_to_all_vulns/up.sql +++ b/lunatrace/bsl/hasura/migrations/lunatrace/1674685930023_add_cve_id_to_all_vulns/up.sql @@ -2,6 +2,6 @@ ALTER TABLE vulnerability.vulnerability ADD COLUMN cve_id text NULL DEFAULT NULL -DROP FUNCTION vulnerability.cisa_known_exploited_vulnerability; +DROP FUNCTION IF EXISTS vulnerability.cisa_known_exploited_vulnerability; -DROP FUNCTION vulnerability.vulnerability_cisa_known_exploited +DROP FUNCTION IF EXISTS vulnerability.vulnerability_cisa_known_exploited diff --git a/lunatrace/npm-package-cli/package.json b/lunatrace/npm-package-cli/package.json index c351c73ff..804bd4a64 100644 --- a/lunatrace/npm-package-cli/package.json +++ b/lunatrace/npm-package-cli/package.json @@ -25,11 +25,18 @@ "@oclif/core": "^2", "@oclif/plugin-help": "^5", "@oclif/plugin-plugins": "^2.2.4", + "@octokit/core": "^4.2.0", + "@octokit/plugin-paginate-rest": "^6.0.0", + "@octokit/rest": "^19.0.7", "npm-package-arg": "^10.1.0", - "pacote": "^15.0.8" + "octokit-plugin-create-pull-request": "^4.1.1", + "pacote": "^15.0.8", + "js-yaml": "3.14." }, "devDependencies": { "@oclif/test": "^2.3.3", + "@octokit/plugin-rest-endpoint-methods": "^7.0.1", + "@octokit/types": "^9.0.0", "@types/chai": "^4", "@types/jest": "^29.4.0", "@types/node": "^16.18.11", diff --git a/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts new file mode 100644 index 000000000..95175b116 --- /dev/null +++ b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2023 by LunaSec (owned by Refinery Labs, Inc) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import { Args, Command, Flags } from '@oclif/core'; + +import { PullRequestOctokit, replacePackageAndFileGitHubPullRequest } from '../../package/github-pr'; +import { replacePackagesForNode } from '../../package/replace-package'; +import { setupPackageTree } from '../../package/replace-package/package-tree'; +import { PackageManagerType } from '../../package/types'; +import { getScriptPath } from '../../package/utils/get-script-path'; +import { ReplacePackageFlags } from '../replace-package'; + +export default class GitHubReplacePackage extends Command { + static description = 'Prints an NPM package tree'; + + static examples = [`$ lunatrace-npm-cli show-tree /path/to/node/project`]; + + static flags = { + ...ReplacePackageFlags, + githubToken: Flags.string({ + aliases: ['github-token'], + env: 'GITHUB_TOKEN', + required: true, + summary: 'GitHub Token to authenticate against the API with.', + }), + gitRef: Flags.string({ + aliases: ['ref'], + required: false, + summary: 'Optional git ref to use for the pull request', + }), + manifestPath: Flags.string({ + aliases: ['manifest-path'], + default: '/', + required: false, + summary: 'Path in the repo to the folder containing manifest files.', + }), + packageManager: Flags.string({ + aliases: ['package-manager'], + default: 'npm', + options: ['npm', 'yarn'], + required: false, + summary: `Package manager to read files from the repo and modify. + If this is npm, it will read 'package-lock.json' and for Yarn 'yarn.lock'.`, + }), + tempWritePath: Flags.string({ + aliases: ['temp-write-path'], + required: false, + summary: 'The folder where the temporary files will be downloaded to and written.', + }), + }; + + static args = { + repo: Args.string({ + description: 'Repo to read package data from. Specified as <:user>/<:repo> (like lunasec-io/lunasec).', + required: true, + }), + }; + + async run(): Promise { + const { args, flags } = await this.parse(GitHubReplacePackage); + + const splitRepo = args.repo.split('/'); + + if (splitRepo.length !== 2) { + throw new Error(`Invalid repo passed. Must be <:user>/<:repo> (like lunasec-io/lunasec)`); + } + + const userOrOrg = splitRepo[0]; + const repo = splitRepo[1]; + + if (!userOrOrg) { + throw new Error('Missing user or org for command'); + } + + if (!repo) { + throw new Error('Missing repo for command'); + } + + const octokit = new PullRequestOctokit({ + auth: flags.githubToken, + }); + + await replacePackageAndFileGitHubPullRequest( + octokit, + userOrOrg, + repo, + flags.manifestPath, + flags.packageManager as PackageManagerType, + flags.old, + flags.new, + flags.gitRef + ); + } +} diff --git a/lunatrace/npm-package-cli/src/commands/replace-package/index.ts b/lunatrace/npm-package-cli/src/commands/replace-package/index.ts index a98b912b5..c30e1f046 100644 --- a/lunatrace/npm-package-cli/src/commands/replace-package/index.ts +++ b/lunatrace/npm-package-cli/src/commands/replace-package/index.ts @@ -14,29 +14,28 @@ * limitations under the License. * */ -import Arborist from '@npmcli/arborist'; import { Args, Command, Flags } from '@oclif/core'; -import npa from 'npm-package-arg'; -import { manifest } from 'pacote'; -import { setupPackageTree } from '../../package/package-tree'; +import { replacePackagesForNode } from '../../package/replace-package'; +import { setupPackageTree } from '../../package/replace-package/package-tree'; import { getScriptPath } from '../../package/utils/get-script-path'; +export const ReplacePackageFlags = { + old: Flags.string({ description: 'Target Package with Semver range to remove from Package.json', required: true }), + new: Flags.string({ + description: 'New Package with Semver range to use as replacement in Package.json', + required: true, + }), +}; + export default class ReplacePackage extends Command { - static description = 'Prints an NPM package tree'; + static description = 'Replaces an NPM package in the package tree'; static examples = [ - `$ lunatrace-npm-cli show-tree /path/to/node/project -`, + `$ lunatrace-npm-cli replace-package /path/to/node/project --old react@^16.0.0 --new react@^16.2.0`, ]; - static flags = { - old: Flags.string({ description: 'Target Package with Semver range to remove from Package.json', required: true }), - new: Flags.string({ - description: 'New Package with Semver range to use as replacement in Package.json', - required: true, - }), - }; + static flags = ReplacePackageFlags; static args = { root: Args.string({ description: 'Root folder to read package.json from', required: false }), @@ -56,36 +55,14 @@ export default class ReplacePackage extends Command { // TODO: Figure out why Arborist marks everything as "extraneous" in the generated lockfile. const node = await tree.loadVirtualTreeFromRoot(); - const oldPackage = npa(flags.old); - - // TODO: Figure out if this works for `git` packages as well. (It probably doesn't and will require a separate code path) - const nodes = await node.querySelectorAll(`[name=${oldPackage.escapedName}]:semver(${oldPackage.rawSpec})`); - - const resolvedManifest = await manifest(flags.new); - - nodes.map((n) => { - if (!n.parent) { - throw new Error('Unable to remove package for node without a parent'); - } - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - n.package = { - ...resolvedManifest, - resolved: resolvedManifest._resolved, - integrity: resolvedManifest._integrity, - }; - - // These fields are required by Arborist to properly update the lockfile. - n.resolved = resolvedManifest._resolved; - n.integrity = resolvedManifest._integrity; - }); + const { updatedNodes } = await replacePackagesForNode(node, flags.old, flags.new); - this.log(`Updated ${nodes.length} packages`); + this.log(`Updated ${updatedNodes.length} packages`); // This updates the package-lock.json file on disk. // Note: We may actually need to call `tree.reify()` in order to get the transitive dependencies to update. // It's unclear and untested currently. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore await node.meta.save(); diff --git a/lunatrace/npm-package-cli/src/commands/show-tree/index.ts b/lunatrace/npm-package-cli/src/commands/show-tree/index.ts index cd4e9a0f8..2c65949bd 100644 --- a/lunatrace/npm-package-cli/src/commands/show-tree/index.ts +++ b/lunatrace/npm-package-cli/src/commands/show-tree/index.ts @@ -19,7 +19,7 @@ import { join } from 'path'; import Arborist from '@npmcli/arborist'; import { Args, Command } from '@oclif/core'; -import { setupPackageTree } from '../../package/package-tree'; +import { setupPackageTree } from '../../package/replace-package/package-tree'; import { getScriptPath } from '../../package/utils/get-script-path'; export default class ShowTree extends Command { diff --git a/lunatrace/npm-package-cli/src/package/github-pr/index.ts b/lunatrace/npm-package-cli/src/package/github-pr/index.ts new file mode 100644 index 000000000..680ebfa35 --- /dev/null +++ b/lunatrace/npm-package-cli/src/package/github-pr/index.ts @@ -0,0 +1,225 @@ +/* + * Copyright 2023 by LunaSec (owned by Refinery Labs, Inc) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import fs from 'fs'; + +import type { Constructor } from '@octokit/core/dist-types/types'; +import type { PaginateInterface } from '@octokit/plugin-paginate-rest'; +import type { RestEndpointMethods } from '@octokit/plugin-rest-endpoint-methods/dist-types/generated/method-types'; +import type { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types'; +import { Octokit } from '@octokit/rest'; +import { createPullRequest } from 'octokit-plugin-create-pull-request'; +import type { Options } from 'octokit-plugin-create-pull-request/dist-types/types'; + +import { replacePackagesForNode } from '../replace-package'; +import { setupPackageTree } from '../replace-package/package-tree'; +import { PackageManagerType } from '../types'; + +export type ManifestFileData = { + manifest: { contents: string; path: string }; + lockfile: { contents: string; path: string }; + packageManagerType: PackageManagerType; +}; + +// type PullRequestOctokitType = typeof Octokit & Constructor<{ createPullRequest: (args_0: Options) => ReturnType }>; +export const PullRequestOctokit = Octokit.plugin(createPullRequest); +export type PullRequestOctokitType = Octokit & { paginate: PaginateInterface } & RestEndpointMethods & + Api & { createPullRequest: (args_0: Options) => Promise }; + +function getManifestLockFilename(packageManager: PackageManagerType): string { + if (packageManager === 'yarn') { + return 'yarn.lock'; + } + return 'package-lock.json'; +} + +async function downloadFileFromGitHub( + octokit: PullRequestOctokitType, + owner: string, + repo: string, + ref: string, + path: string, + filename: string +) { + const normalizedPath = path.replace(/\/$/, ''); + + const response = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', { + owner, + repo, + ref, + path: `${normalizedPath}/${filename}`, + }); + + if (response.status !== 200) { + throw new Error(`GitHub Error: Unable to download ${filename} from ${owner}/${repo}/${ref}/${path}`); + } + + if (Array.isArray(response.data)) { + throw new Error(`GitHub Error: ${owner}/${repo}/${ref}/${path}/package.json is a directory.`); + } + + if (response.data.type !== 'file') { + throw new Error(`GitHub Error: ${owner}/${repo}/${ref}/${path}/package.json is not a file.`); + } + + return { + contents: response.data.content, + path: response.data.path, + }; +} + +export async function downloadManifestFromGithub( + octokit: PullRequestOctokitType, + owner: string, + repo: string, + ref: string, + path: string, + packageManager: PackageManagerType +): Promise { + const packageJson = await downloadFileFromGitHub(octokit, owner, repo, ref, path, 'package.json'); + + const lockfile = await downloadFileFromGitHub( + octokit, + owner, + repo, + ref, + path, + getManifestLockFilename(packageManager) + ); + + return { + manifest: packageJson, + lockfile: lockfile, + packageManagerType: packageManager, + }; +} + +export async function writeManifestToFolder(manifestFileData: ManifestFileData, directory: string): Promise { + const packageJsonPath = `${directory}/package.json`; + const packageLockPath = `${directory}/${getManifestLockFilename(manifestFileData.packageManagerType)}`; + + await fs.promises.mkdir(directory, { recursive: true }); + await fs.promises.writeFile(packageJsonPath, manifestFileData.manifest.contents, 'base64'); + await fs.promises.writeFile(packageLockPath, manifestFileData.lockfile.contents, 'base64'); +} + +export async function getRepoAuthState( + octokit: PullRequestOctokitType, + owner: string, + repo: string +): Promise<{ + isUser: boolean; + baseBranch: string; + // TODO: Make this actually have a real type. + repository: any; +}> { + const { data: repository, headers } = await octokit.request('GET /repos/{owner}/{repo}', { + owner, + repo, + }); + + const isUser = !!headers['x-oauth-scopes']; + + if (!repository.permissions) { + throw new Error(`GitHub Error: Missing auth for repo ${owner}/${repo}`); + } + + return { isUser, baseBranch: repository.default_branch, repository }; +} + +export async function replacePackageAndFileGitHubPullRequest( + octokit: PullRequestOctokitType, + owner: string, + repo: string, + path: string, + packageManager: PackageManagerType, + oldPackage: string, + newPackage: string, + ref?: string +): Promise { + const repoAuthState = await getRepoAuthState(octokit, owner, repo); + + const checkoutRef = ref || repoAuthState.baseBranch; + + const manifestFileData = await downloadManifestFromGithub(octokit, owner, repo, checkoutRef, path, packageManager); + + const tmpDirPath = '/tmp/manifest' + Math.round(Math.random() * 1000000).toString(10); + + await writeManifestToFolder(manifestFileData, tmpDirPath); + + const tree = setupPackageTree({ + root: tmpDirPath, + packageManager: packageManager, + }); + + // TODO: Figure out why Arborist marks everything as "extraneous" in the generated lockfile. + const node = await tree.loadVirtualTreeFromRoot(); + + const { updatedNodes } = await replacePackagesForNode(node, oldPackage, newPackage); + + // TODO: Replace this with a logger. + console.log(`Updated ${updatedNodes.length} packages`); + + // This updates the package-lock.json file on disk. + // Note: We may actually need to call `tree.reify()` in order to get the transitive dependencies to update. + // It's unclear and untested currently. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + await node.meta.save(); + + // Read new manifest and lockfile contents + const newManifestData = await fs.promises.readFile(`${tmpDirPath}/package.json`, 'utf8'); + const newLockfileData = await fs.promises.readFile( + `${tmpDirPath}/${getManifestLockFilename(packageManager)}`, + 'utf8' + ); + + const normalizedOldPackageName = oldPackage.replace(/[\W]+/g, ''); + const normalizedNewPackageName = newPackage.replace(/[\W]+/g, ''); + + const branchName = `replace-${normalizedOldPackageName}-with-${normalizedNewPackageName}`; + const message = `This pull request replaces \`${oldPackage}\` with \`${newPackage}\` in \`${owner}/${repo}@${checkoutRef}\` in path: \`${path}\``; + + const pullRequest = await octokit.createPullRequest({ + owner, + repo, + title: `[Security][LunaTrace] Replace ${oldPackage} with ${newPackage}`, + head: branchName, + base: ref, + createWhenEmpty: false, + body: message, + changes: { + files: { + [manifestFileData.manifest.path]: newManifestData, + [manifestFileData.lockfile.path]: newLockfileData, + }, + commit: message, + author: { + name: 'LunaTrace', + email: 'github-bot@lunasec.io', + date: new Date().toISOString(), + }, + // TODO: Figure out if we want to allow for a custom committer. + // TODO: Figure out signed commits. + }, + }); + + console.log('PR Created:', pullRequest.data.html_url); + console.log('PR Title:', pullRequest.data.title); + + // Delete temporary directory + await fs.promises.rm(tmpDirPath, { recursive: true, force: true }); +} diff --git a/lunatrace/npm-package-cli/src/package/replace-package/index.ts b/lunatrace/npm-package-cli/src/package/replace-package/index.ts new file mode 100644 index 000000000..ffb5b8d83 --- /dev/null +++ b/lunatrace/npm-package-cli/src/package/replace-package/index.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2023 by LunaSec (owned by Refinery Labs, Inc) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// noinspection CssInvalidPseudoSelector JSConstantReassignment + +import { Node } from '@npmcli/arborist'; +import npa, { Result } from 'npm-package-arg'; +import { AbbreviatedManifest, manifest, ManifestResult } from 'pacote'; + +export async function replacePackagesForNode( + node: Node, + oldPackage: string, + newPackage: string +): Promise<{ updatedNodes: Node[]; newPackageManifest: AbbreviatedManifest & ManifestResult }> { + const { escapedName, rawSpec } = npa(oldPackage); + + // TODO: Figure out if this works for `git` packages as well. (It probably doesn't and will require a separate code path) + const nodes = await node.querySelectorAll(`[name=${escapedName}]:semver(${rawSpec})`); + + const newPackageManifest = await manifest(newPackage); + + nodes.map((n) => { + if (!n.parent) { + throw new Error('Unable to remove package for node without a parent'); + } + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + n.package = { + ...newPackageManifest, + resolved: newPackageManifest._resolved, + integrity: newPackageManifest._integrity, + }; + + // These fields are required by Arborist to properly update the lockfile. + n.resolved = newPackageManifest._resolved; + n.integrity = newPackageManifest._integrity; + }); + + return { + updatedNodes: nodes, + newPackageManifest, + }; +} diff --git a/lunatrace/npm-package-cli/src/package/package-tree.ts b/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts similarity index 89% rename from lunatrace/npm-package-cli/src/package/package-tree.ts rename to lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts index c952eb226..cdac026a8 100644 --- a/lunatrace/npm-package-cli/src/package/package-tree.ts +++ b/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts @@ -18,7 +18,7 @@ import { join } from 'path'; import Arborist, { Node } from '@npmcli/arborist'; -import { PackageTreeConfig } from './types'; +import { PackageTreeConfig } from '../types'; function getPackageJsonPath(config: PackageTreeConfig): string { return config.packageJsonPath || join(config.root, 'package.json'); @@ -71,15 +71,11 @@ export class PackageTree { this.fullPackageJsonPath = config.fullPackageJsonPath; this.fullPackageLockPath = config.fullPackageLockPath; this.root = config.root; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - this.arborist = this.createArborist({ - packageLockOnly: true, - }); - } - - createArborist() { - return new Arborist({ + this.arborist = new Arborist({ path: this.root, + packageLockOnly: true, }); } @@ -96,10 +92,4 @@ export class PackageTree { return this.arborist.loadVirtual({ root: root }); } - - async loadPackageNode(packageSlug: string): Promise { - const tree = await this.loadVirtualTreeFromRoot(); - - return tree.querySelectorAll(packageSlug); - } } diff --git a/yarn.lock b/yarn.lock index 7f59cfb61..0e3aac5b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13405,6 +13405,11 @@ __metadata: "@oclif/plugin-help": ^5 "@oclif/plugin-plugins": ^2.2.4 "@oclif/test": ^2.3.3 + "@octokit/core": ^4.2.0 + "@octokit/plugin-paginate-rest": ^6.0.0 + "@octokit/plugin-rest-endpoint-methods": ^7.0.1 + "@octokit/rest": ^19.0.7 + "@octokit/types": ^9.0.0 "@types/chai": ^4 "@types/jest": ^29.4.0 "@types/node": ^16.18.11 @@ -13417,6 +13422,7 @@ __metadata: jest: ^29.4.1 npm-package-arg: ^10.1.0 oclif: ^3.6.1 + octokit-plugin-create-pull-request: ^4.1.1 pacote: ^15.0.8 shx: ^0.3.4 ts-jest: ^29.0.5 @@ -14887,6 +14893,15 @@ __metadata: languageName: node linkType: hard +"@octokit/auth-token@npm:^3.0.0": + version: 3.0.3 + resolution: "@octokit/auth-token@npm:3.0.3" + dependencies: + "@octokit/types": ^9.0.0 + checksum: 9b3f569cec1b7e0aa88ab6da68aed4b49b6652261bd957257541fabaf6a4d4ed99f908153cc3dd2fe15b8b0ccaff8caaafaa50bb1a4de3925b0954a47cca1900 + languageName: node + linkType: hard + "@octokit/auth-unauthenticated@npm:^2.0.0, @octokit/auth-unauthenticated@npm:^2.0.4": version: 2.1.0 resolution: "@octokit/auth-unauthenticated@npm:2.1.0" @@ -14927,6 +14942,21 @@ __metadata: languageName: node linkType: hard +"@octokit/core@npm:^4.1.0, @octokit/core@npm:^4.2.0": + version: 4.2.0 + resolution: "@octokit/core@npm:4.2.0" + dependencies: + "@octokit/auth-token": ^3.0.0 + "@octokit/graphql": ^5.0.0 + "@octokit/request": ^6.0.0 + "@octokit/request-error": ^3.0.0 + "@octokit/types": ^9.0.0 + before-after-hook: ^2.2.0 + universal-user-agent: ^6.0.0 + checksum: 5ac56e7f14b42a5da8d3075a2ae41483521a78bee061a01f4a81d8c0ecd6a684b2e945d66baba0cd1fdf264639deedc3a96d0f32c4d2fc39b49ca10f52f4de39 + languageName: node + linkType: hard + "@octokit/endpoint@npm:^6.0.1": version: 6.0.12 resolution: "@octokit/endpoint@npm:6.0.12" @@ -14938,6 +14968,17 @@ __metadata: languageName: node linkType: hard +"@octokit/endpoint@npm:^7.0.0": + version: 7.0.5 + resolution: "@octokit/endpoint@npm:7.0.5" + dependencies: + "@octokit/types": ^9.0.0 + is-plain-object: ^5.0.0 + universal-user-agent: ^6.0.0 + checksum: 81c9e9eabf50e48940cceff7c4d7fbc9327190296507cfe8a199ea00cd492caf8f18a841caf4e3619828924b481996eb16091826db6b5a649bee44c8718ecaa9 + languageName: node + linkType: hard + "@octokit/graphql@npm:^4.5.8, @octokit/graphql@npm:^4.8.0": version: 4.8.0 resolution: "@octokit/graphql@npm:4.8.0" @@ -14949,6 +14990,17 @@ __metadata: languageName: node linkType: hard +"@octokit/graphql@npm:^5.0.0": + version: 5.0.5 + resolution: "@octokit/graphql@npm:5.0.5" + dependencies: + "@octokit/request": ^6.0.0 + "@octokit/types": ^9.0.0 + universal-user-agent: ^6.0.0 + checksum: eb2d1a6305a3d1f55ff0ce92fb88b677f0bb789757152d58a79ef61171fb65ecf6fe18d6c27e236c0cee6a0c2600c2cb8370f5ac7184f8e9361c085aa4555bb1 + languageName: node + linkType: hard + "@octokit/oauth-app@npm:^3.3.2, @octokit/oauth-app@npm:^3.5.1": version: 3.6.0 resolution: "@octokit/oauth-app@npm:3.6.0" @@ -15000,6 +15052,20 @@ __metadata: languageName: node linkType: hard +"@octokit/openapi-types@npm:^14.0.0": + version: 14.0.0 + resolution: "@octokit/openapi-types@npm:14.0.0" + checksum: 0a1f8f3be998cd82c5a640e9166d43fd183b33d5d36f5e1a9b81608e94d0da87c01ec46c9988f69cd26585d4e2ffc4d3ec99ee4f75e5fe997fc86dad0aa8293c + languageName: node + linkType: hard + +"@octokit/openapi-types@npm:^16.0.0": + version: 16.0.0 + resolution: "@octokit/openapi-types@npm:16.0.0" + checksum: 844f30a545da380d63c712e0eb733366bc567d1aab34529c79fdfbec3d73810e81d83f06fdab13058a5cbc7dae786db1a9b90b5b61b1e606854ee45d5ec5f194 + languageName: node + linkType: hard + "@octokit/plugin-enterprise-rest@npm:^6.0.1": version: 6.0.1 resolution: "@octokit/plugin-enterprise-rest@npm:6.0.1" @@ -15027,6 +15093,17 @@ __metadata: languageName: node linkType: hard +"@octokit/plugin-paginate-rest@npm:^6.0.0": + version: 6.0.0 + resolution: "@octokit/plugin-paginate-rest@npm:6.0.0" + dependencies: + "@octokit/types": ^9.0.0 + peerDependencies: + "@octokit/core": ">=4" + checksum: 4ad89568d883373898b733837cada7d849d51eef32157c11d4a81cef5ce8e509720d79b46918cada3c132f9b29847e383f17b7cd5c39ede7c93cdcd2f850b47f + languageName: node + linkType: hard + "@octokit/plugin-request-log@npm:^1.0.0, @octokit/plugin-request-log@npm:^1.0.4": version: 1.0.4 resolution: "@octokit/plugin-request-log@npm:1.0.4" @@ -15058,6 +15135,18 @@ __metadata: languageName: node linkType: hard +"@octokit/plugin-rest-endpoint-methods@npm:^7.0.0, @octokit/plugin-rest-endpoint-methods@npm:^7.0.1": + version: 7.0.1 + resolution: "@octokit/plugin-rest-endpoint-methods@npm:7.0.1" + dependencies: + "@octokit/types": ^9.0.0 + deprecation: ^2.3.1 + peerDependencies: + "@octokit/core": ">=3" + checksum: cdb8734ec960f75acc2405284920c58efac9a71b1c3b2a71662b9100ffbc22dac597150acff017a93459c57e9a492d9e1c27872b068387dbb90597de75065fcf + languageName: node + linkType: hard + "@octokit/plugin-retry@npm:^3.0.9": version: 3.0.9 resolution: "@octokit/plugin-retry@npm:3.0.9" @@ -15127,6 +15216,20 @@ __metadata: languageName: node linkType: hard +"@octokit/request@npm:^6.0.0": + version: 6.2.3 + resolution: "@octokit/request@npm:6.2.3" + dependencies: + "@octokit/endpoint": ^7.0.0 + "@octokit/request-error": ^3.0.0 + "@octokit/types": ^9.0.0 + is-plain-object: ^5.0.0 + node-fetch: ^2.6.7 + universal-user-agent: ^6.0.0 + checksum: fef4097be8375d20bb0b3276d8a3adf866ec628f2b0664d334f3c29b92157da847899497abdc7a5be540053819b55564990543175ad48f04e9e6f25f0395d4d3 + languageName: node + linkType: hard + "@octokit/rest@npm:^16.28.4": version: 16.43.2 resolution: "@octokit/rest@npm:16.43.2" @@ -15163,6 +15266,18 @@ __metadata: languageName: node linkType: hard +"@octokit/rest@npm:^19.0.7": + version: 19.0.7 + resolution: "@octokit/rest@npm:19.0.7" + dependencies: + "@octokit/core": ^4.1.0 + "@octokit/plugin-paginate-rest": ^6.0.0 + "@octokit/plugin-request-log": ^1.0.4 + "@octokit/plugin-rest-endpoint-methods": ^7.0.0 + checksum: 1f970c4de2cf3d1691d3cf5dd4bfa5ac205629e76417b5c51561e1beb5b4a7e6c65ba647f368727e67e5243418e06ca9cdafd9e733173e1529385d4f4d053d3d + languageName: node + linkType: hard + "@octokit/types@npm:^2.0.0, @octokit/types@npm:^2.0.1": version: 2.16.2 resolution: "@octokit/types@npm:2.16.2" @@ -15190,6 +15305,24 @@ __metadata: languageName: node linkType: hard +"@octokit/types@npm:^8.0.0": + version: 8.2.1 + resolution: "@octokit/types@npm:8.2.1" + dependencies: + "@octokit/openapi-types": ^14.0.0 + checksum: 92f2fe5ea8c4c6ddbb2363c74cd865c64e5753eaa4895bc925b5064390890b1441c5406015d8a92285f386cc7e6fe714c47fe4beda370fcda9177153299c9e37 + languageName: node + linkType: hard + +"@octokit/types@npm:^9.0.0": + version: 9.0.0 + resolution: "@octokit/types@npm:9.0.0" + dependencies: + "@octokit/openapi-types": ^16.0.0 + checksum: 5c7f5cca8f00f7c4daa0d00f4fe991c1598ec47cd6ced50b1c5fbe9721bb9dee0adc2acdee265a3a715bb984e53ef3dc7f1cfb7326f712c6d809d59fc5c6648d + languageName: node + linkType: hard + "@octokit/webhooks-methods@npm:^2.0.0": version: 2.0.0 resolution: "@octokit/webhooks-methods@npm:2.0.0" @@ -43929,6 +44062,15 @@ __metadata: languageName: node linkType: hard +"octokit-plugin-create-pull-request@npm:^4.1.1": + version: 4.1.1 + resolution: "octokit-plugin-create-pull-request@npm:4.1.1" + dependencies: + "@octokit/types": ^8.0.0 + checksum: 889d578b43befcc46c6bfb774a87e12b99d70fa798764346587995257e2e5404b9fbb25a267708982ce9559abd80b9ecbf9537b973b081281ecc6171624e0130 + languageName: node + linkType: hard + "octokit@npm:^1.7.1": version: 1.7.1 resolution: "octokit@npm:1.7.1" From d918e427c08ffbd036caeb94292dcb908d461302 Mon Sep 17 00:00:00 2001 From: Free Wortley Date: Wed, 15 Feb 2023 18:17:59 -0800 Subject: [PATCH 2/4] Fix busted up stuff --- lunatrace/npm-package-cli/package.json | 3 +-- .../src/commands/github-pr/replace-package.ts | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lunatrace/npm-package-cli/package.json b/lunatrace/npm-package-cli/package.json index 804bd4a64..9b33608d1 100644 --- a/lunatrace/npm-package-cli/package.json +++ b/lunatrace/npm-package-cli/package.json @@ -30,8 +30,7 @@ "@octokit/rest": "^19.0.7", "npm-package-arg": "^10.1.0", "octokit-plugin-create-pull-request": "^4.1.1", - "pacote": "^15.0.8", - "js-yaml": "3.14." + "pacote": "^15.0.8" }, "devDependencies": { "@oclif/test": "^2.3.3", diff --git a/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts index 95175b116..c1803d54c 100644 --- a/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts +++ b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts @@ -24,9 +24,11 @@ import { getScriptPath } from '../../package/utils/get-script-path'; import { ReplacePackageFlags } from '../replace-package'; export default class GitHubReplacePackage extends Command { - static description = 'Prints an NPM package tree'; + static description = 'Replaces a package in an NPM project on GitHub and then files a Pull Request'; - static examples = [`$ lunatrace-npm-cli show-tree /path/to/node/project`]; + static examples = [ + `$ lunatrace-npm-cli github-pr replace-package owner/repo --githubToken --old js-yaml@^3.13.1 --new js-yaml@^3.14.0`, + ]; static flags = { ...ReplacePackageFlags, From 0263bca06c34af5e57d0f253156f028bb989e28c Mon Sep 17 00:00:00 2001 From: Free Wortley Date: Thu, 16 Feb 2023 12:53:23 -0800 Subject: [PATCH 3/4] Fix bug with Arborist marking packages as extraneous --- .../src/commands/replace-package/index.ts | 2 +- .../src/commands/show-tree/index.ts | 2 +- .../npm-package-cli/src/package/github-pr/index.ts | 3 ++- .../src/package/replace-package/package-tree.ts | 14 -------------- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/lunatrace/npm-package-cli/src/commands/replace-package/index.ts b/lunatrace/npm-package-cli/src/commands/replace-package/index.ts index c30e1f046..bb52f19cc 100644 --- a/lunatrace/npm-package-cli/src/commands/replace-package/index.ts +++ b/lunatrace/npm-package-cli/src/commands/replace-package/index.ts @@ -53,7 +53,7 @@ export default class ReplacePackage extends Command { }); // TODO: Figure out why Arborist marks everything as "extraneous" in the generated lockfile. - const node = await tree.loadVirtualTreeFromRoot(); + const node = await tree.arborist.loadVirtual(); const { updatedNodes } = await replacePackagesForNode(node, flags.old, flags.new); diff --git a/lunatrace/npm-package-cli/src/commands/show-tree/index.ts b/lunatrace/npm-package-cli/src/commands/show-tree/index.ts index 2c65949bd..ce66fed48 100644 --- a/lunatrace/npm-package-cli/src/commands/show-tree/index.ts +++ b/lunatrace/npm-package-cli/src/commands/show-tree/index.ts @@ -45,7 +45,7 @@ export default class ShowTree extends Command { root: root, }); - const virtualTree = await tree.loadVirtualTreeFromRoot(); + const virtualTree = await tree.arborist.loadVirtual(); this.log('Package tree:\n'); diff --git a/lunatrace/npm-package-cli/src/package/github-pr/index.ts b/lunatrace/npm-package-cli/src/package/github-pr/index.ts index 680ebfa35..ced00e454 100644 --- a/lunatrace/npm-package-cli/src/package/github-pr/index.ts +++ b/lunatrace/npm-package-cli/src/package/github-pr/index.ts @@ -166,7 +166,7 @@ export async function replacePackageAndFileGitHubPullRequest( }); // TODO: Figure out why Arborist marks everything as "extraneous" in the generated lockfile. - const node = await tree.loadVirtualTreeFromRoot(); + const node = await tree.arborist.loadVirtual(); const { updatedNodes } = await replacePackagesForNode(node, oldPackage, newPackage); @@ -201,6 +201,7 @@ export async function replacePackageAndFileGitHubPullRequest( base: ref, createWhenEmpty: false, body: message, + update: true, changes: { files: { [manifestFileData.manifest.path]: newManifestData, diff --git a/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts b/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts index cdac026a8..f54599910 100644 --- a/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts +++ b/lunatrace/npm-package-cli/src/package/replace-package/package-tree.ts @@ -78,18 +78,4 @@ export class PackageTree { packageLockOnly: true, }); } - - /** - * Loads the entire package tree from the root node. - */ - loadVirtualTreeFromRoot(): Promise { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const root = new Node({ - path: this.root, - pkg: this.fullPackageJsonPath, - }); - - return this.arborist.loadVirtual({ root: root }); - } } From 6da258a221d46e29fcc868da29cfa3f84a328972 Mon Sep 17 00:00:00 2001 From: Free Wortley Date: Thu, 16 Feb 2023 13:30:40 -0800 Subject: [PATCH 4/4] Make types less janky --- .../src/commands/github-pr/replace-package.ts | 9 +++--- .../src/package/github-pr/index.ts | 29 ++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts index c1803d54c..a04e32ef6 100644 --- a/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts +++ b/lunatrace/npm-package-cli/src/commands/github-pr/replace-package.ts @@ -16,11 +16,12 @@ */ import { Args, Command, Flags } from '@oclif/core'; -import { PullRequestOctokit, replacePackageAndFileGitHubPullRequest } from '../../package/github-pr'; -import { replacePackagesForNode } from '../../package/replace-package'; -import { setupPackageTree } from '../../package/replace-package/package-tree'; +import { + PullRequestOctokit, + PullRequestOctokitType, + replacePackageAndFileGitHubPullRequest, +} from '../../package/github-pr'; import { PackageManagerType } from '../../package/types'; -import { getScriptPath } from '../../package/utils/get-script-path'; import { ReplacePackageFlags } from '../replace-package'; export default class GitHubReplacePackage extends Command { diff --git a/lunatrace/npm-package-cli/src/package/github-pr/index.ts b/lunatrace/npm-package-cli/src/package/github-pr/index.ts index ced00e454..2298798db 100644 --- a/lunatrace/npm-package-cli/src/package/github-pr/index.ts +++ b/lunatrace/npm-package-cli/src/package/github-pr/index.ts @@ -16,7 +16,6 @@ */ import fs from 'fs'; -import type { Constructor } from '@octokit/core/dist-types/types'; import type { PaginateInterface } from '@octokit/plugin-paginate-rest'; import type { RestEndpointMethods } from '@octokit/plugin-rest-endpoint-methods/dist-types/generated/method-types'; import type { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types'; @@ -34,10 +33,12 @@ export type ManifestFileData = { packageManagerType: PackageManagerType; }; -// type PullRequestOctokitType = typeof Octokit & Constructor<{ createPullRequest: (args_0: Options) => ReturnType }>; export const PullRequestOctokit = Octokit.plugin(createPullRequest); + +// This type is annoying but without it, we don't seem to get proper type checking for the PR plugin call. export type PullRequestOctokitType = Octokit & { paginate: PaginateInterface } & RestEndpointMethods & - Api & { createPullRequest: (args_0: Options) => Promise }; + Api & + ReturnType; function getManifestLockFilename(packageManager: PackageManagerType): string { if (packageManager === 'yarn') { @@ -46,6 +47,10 @@ function getManifestLockFilename(packageManager: PackageManagerType): string { return 'package-lock.json'; } +/** + * Makes a request to GitHub to download a file from a repo with the specific ref (commit hash). + * The path is merged with the filename to create the absolute path to the file. + */ async function downloadFileFromGitHub( octokit: PullRequestOctokitType, owner: string, @@ -126,11 +131,17 @@ export async function getRepoAuthState( // TODO: Make this actually have a real type. repository: any; }> { - const { data: repository, headers } = await octokit.request('GET /repos/{owner}/{repo}', { + const response = await octokit.request('GET /repos/{owner}/{repo}', { owner, repo, }); + if (response.status !== 200) { + throw new Error(`GitHub Error: Unable to get repo ${owner}/${repo} (Status: ${response.status})`); + } + + const { data: repository, headers } = response; + const isUser = !!headers['x-oauth-scopes']; if (!repository.permissions) { @@ -218,6 +229,16 @@ export async function replacePackageAndFileGitHubPullRequest( }, }); + if (!pullRequest) { + throw new Error(`GitHub Error: Unable to create pull request for ${owner}/${repo}@${checkoutRef} in path: ${path}`); + } + + if (pullRequest.status !== 200) { + throw new Error( + `GitHub Error: Unable to create pull request for ${owner}/${repo}@${checkoutRef} in path: ${path} (Status: ${pullRequest.status})` + ); + } + console.log('PR Created:', pullRequest.data.html_url); console.log('PR Title:', pullRequest.data.title);