Merge pull request #1019 from ckeditor/i/3828-restartable-release

Feature (ci): Created a new binary script called `ckeditor5-dev-ci-is-workflow-restarted` that returns with a non-zero exit code if a given workflow is executed for the first time. The restarted workflows exit with a zero exit code.

Feature (release-tools): A user-provided version will be checked against npm availability while generating a changelog. If it is already taken, the tools will not allow it to be used.

Other (release-tools): The `updateVersions()` task will no longer verify if the specified `version` is available on npm.

Other (release-tools): The `publishPackages()` task filters out already published packages to avoid pushing the same archive twice. Thanks to that, it can be a part of a process that would be restarted.

Other (release-tools): The `publishPackages()` task tries to publish the package once again when it fails independently from the returned error code. Previously, it was scheduled only when the `E409` error occurred.

Other (release-tools): The `verifyPackagesPublishedCorrectly()` task is no longer available as its responsibility has been merged into the `publishPackages()` task.

Other (release-tools): The `commitAndTag()` task does not commit files if a tag for the specified version is already created.

package.json

@@ -29,6 +29,7 @@
     "lint-staged": "^15.0.0",
     "listr2": "^8.0.0",
     "minimist": "^1.2.8",
+    "simple-git": "^3.27.0",
     "semver": "^7.6.3",
     "upath": "^2.0.1"
   },

packages/ckeditor5-dev-ci/bin/is-workflow-restarted.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+/* eslint-env node */
+
+/**
+ * This script checks if the provided workflow has been restarted. If so, it exits with a zero error code.
+ *
+ * In order to integrate the action in your pipeline, you need prepare a few environment variables:
+ *
+ *   - CIRCLE_WORKFLOW_ID - provided by default by CircleCI and keeps the workflow id.
+ *   - CKE5_CIRCLE_TOKEN - an authorization token to talk to CircleCI REST API.
+ *
+ * Example usage:
+ * CKE5_CIRCLE_TOKEN=... ckeditor5-dev-ci-is-workflow-restarted
+ */
+
+const {
+	CKE5_CIRCLE_TOKEN,
+	CIRCLE_WORKFLOW_ID
+} = process.env;
+
+const requestUrl = `https://circleci.com/api/v2/workflow/${ CIRCLE_WORKFLOW_ID }`;
+
+const requestOptions = {
+	method: 'GET',
+	headers: {
+		'Circle-Token': CKE5_CIRCLE_TOKEN
+	}
+};
+
+fetch( requestUrl, requestOptions )
+	.then( res => res.json() )
+	.then( response => {
+		const { tag = '' } = response;
+
+		if ( tag.startsWith( 'rerun' ) ) {
+			return process.exit( 0 );
+		}
+
+		return process.exit( 1 );
+	} );

packages/ckeditor5-dev-ci/package.json

@@ -29,6 +29,7 @@
     "ckeditor5-dev-ci-allocate-swap-memory": "bin/allocate-swap-memory.sh",
     "ckeditor5-dev-ci-install-latest-chrome": "bin/install-latest-chrome.sh",
     "ckeditor5-dev-ci-is-job-triggered-by-member": "bin/is-job-triggered-by-member.js",
+    "ckeditor5-dev-ci-is-workflow-restarted": "bin/is-workflow-restarted.js",
     "ckeditor5-dev-ci-trigger-circle-build": "bin/trigger-circle-build.js",
     "ckeditor5-dev-ci-circle-disable-auto-cancel-builds": "bin/circle-disable-auto-cancel-builds.js",
     "ckeditor5-dev-ci-circle-enable-auto-cancel-builds": "bin/circle-enable-auto-cancel-builds.js"

packages/ckeditor5-dev-release-tools/lib/index.js

@@ -29,7 +29,7 @@ export { default as saveChangelog } from './utils/savechangelog.js';
 export { default as executeInParallel } from './utils/executeinparallel.js';
 export { default as validateRepositoryToRelease } from './utils/validaterepositorytorelease.js';
 export { default as checkVersionAvailability } from './utils/checkversionavailability.js';
-export { default as verifyPackagesPublishedCorrectly } from './tasks/verifypackagespublishedcorrectly.js';
 export { default as getNpmTagFromVersion } from './utils/getnpmtagfromversion.js';
 export { default as isVersionPublishableForTag } from './utils/isversionpublishablefortag.js';
 export { default as provideToken } from './utils/providetoken.js';
+export { default as findPathsToPackages } from './utils/findpathstopackages.js';

packages/ckeditor5-dev-release-tools/lib/tasks/cleanuppackages.js

@@ -6,6 +6,7 @@
 import fs from 'fs-extra';
 import upath from 'upath';
 import { glob } from 'glob';
+import findPathsToPackages from '../utils/findpathstopackages.js';
 
 /**
  * The purpose of the script is to clean all packages prepared for the release. The cleaning consists of two stages:
@@ -29,12 +30,7 @@ import { glob } from 'glob';
  */
 export default async function cleanUpPackages( options ) {
 	const { packagesDirectory, packageJsonFieldsToRemove, preservePostInstallHook, cwd } = parseOptions( options );
-
-	const packageJsonPaths = await glob( '*/package.json', {
-		cwd: upath.join( cwd, packagesDirectory ),
-		nodir: true,
-		absolute: true
-	} );
+	const packageJsonPaths = await findPathsToPackages( cwd, packagesDirectory, { includePackageJson: true } );
 
 	for ( const packageJsonPath of packageJsonPaths ) {
 		const packagePath = upath.dirname( packageJsonPath );

packages/ckeditor5-dev-release-tools/lib/tasks/commitandtag.js

@@ -4,9 +4,8 @@
  */
 
 import upath from 'upath';
-import { tools } from '@ckeditor/ckeditor5-dev-utils';
 import { glob } from 'glob';
-import shellEscape from 'shell-escape';
+import { simpleGit } from 'simple-git';
 
 const { toUnix } = upath;
 
@@ -27,22 +26,18 @@ export default async function commitAndTag( { version, files, cwd = process.cwd(
 		return;
 	}
 
-	const shExecOptions = {
-		cwd: normalizedCwd,
-		async: true,
-		verbosity: 'silent'
-	};
+	const git = simpleGit( {
+		baseDir: normalizedCwd
+	} );
 
-	// Run the command separately for each file to avoid exceeding the maximum command length on Windows, which is 32767 characters.
-	for ( const filePath of filePathsToAdd ) {
-		await tools.shExec( `git add ${ shellEscape( [ filePath ] ) }`, shExecOptions );
-	}
+	const { all: availableTags } = await git.tags();
+	const tagForVersion = availableTags.find( tag => tag.endsWith( version ) );
 
-	const escapedVersion = {
-		commit: shellEscape( [ `Release: v${ version }.` ] ),
-		tag: shellEscape( [ `v${ version }` ] )
-	};
+	// Do not commit and create tags if a tag is already taken. It might happen when a release job is restarted.
+	if ( tagForVersion ) {
+		return;
+	}
 
-	await tools.shExec( `git commit --message ${ escapedVersion.commit } --no-verify`, shExecOptions );
-	await tools.shExec( `git tag ${ escapedVersion.tag }`, shExecOptions );
+	await git.commit( `Release: v${ version }.`, filePathsToAdd );
+	await git.addTag( `v${ version }` );
 }

packages/ckeditor5-dev-release-tools/lib/tasks/generatechangelogformonorepository.js

@@ -301,7 +301,14 @@ export default async function generateChangelogForMonoRepository( options ) {
 			bumpType = 'patch';
 		}
 
-		return provideNewVersionForMonoRepository( highestVersion, packageHighestVersion, bumpType, { indentLevel: 1 } )
+		const provideVersionOptions = {
+			packageName: packageHighestVersion,
+			version: highestVersion,
+			bumpType,
+			indentLevel: 1
+		};
+
+		return provideNewVersionForMonoRepository( provideVersionOptions )
 			.then( version => {
 				nextVersion = version;
 

packages/ckeditor5-dev-release-tools/lib/tasks/generatechangelogforsinglepackage.js

@@ -79,11 +79,14 @@ export default async function generateChangelogForSinglePackage( options = {} )
 		.then( () => {
 			logProcess( 'Preparing new version for the package...' );
 
-			const releaseType = getNewVersionType( allCommits );
-
 			displayCommits( allCommits, { indentLevel: 1 } );
 
-			return provideVersion( pkgJson.version, releaseType, { indentLevel: 1 } );
+			return provideVersion( {
+				packageName: pkgJson.name,
+				version: pkgJson.version,
+				indentLevel: 1,
+				releaseTypeOrNewVersion: getNewVersionType( allCommits )
+			} );
 		} )
 		.then( version => {
 			if ( version === 'skip' ) {

packages/ckeditor5-dev-release-tools/lib/tasks/publishpackages.js

@@ -4,44 +4,50 @@
  */
 
 import upath from 'upath';
-import { glob } from 'glob';
+import fs from 'fs-extra';
 import assertNpmAuthorization from '../utils/assertnpmauthorization.js';
 import assertPackages from '../utils/assertpackages.js';
 import assertNpmTag from '../utils/assertnpmtag.js';
 import assertFilesToPublish from '../utils/assertfilestopublish.js';
 import executeInParallel from '../utils/executeinparallel.js';
 import publishPackageOnNpmCallback from '../utils/publishpackageonnpmcallback.js';
+import checkVersionAvailability from '../utils/checkversionavailability.js';
+import findPathsToPackages from '../utils/findpathstopackages.js';
 
 /**
  * The purpose of the script is to validate the packages prepared for the release and then release them on npm.
  *
  * The validation contains the following steps in each package:
  * - User must be logged to npm on the specified account.
- * - The package directory mmust contain `package.json` file.
+ * - The package directory must contain `package.json` file.
  * - All other files expected to be released must exist in the package directory.
  * - The npm tag must match the tag calculated from the package version.
  *
  * When the validation for each package passes, packages are published on npm. Optional callback is called for confirmation whether to
  * continue.
  *
+ * If a package has already been published, the script does not try to publish it again. Instead, it treats the package as published.
+ * Whenever a communication between the script and npm fails, it tries to re-publish a package (up to three attempts).
+ *
  * @param {object} options
  * @param {string} options.packagesDirectory Relative path to a location of packages to release.
  * @param {string} options.npmOwner The account name on npm, which should be used to publish the packages.
- * @param {ListrTaskObject} options.listrTask An instance of `ListrTask`.
+ * @param {ListrTaskObject} [options.listrTask] An instance of `ListrTask`.
  * @param {AbortSignal|null} [options.signal=null] Signal to abort the asynchronous process.
  * @param {string} [options.npmTag='staging'] The npm distribution tag.
  * @param {Object.<string, Array.<string>>|null} [options.optionalEntries=null] Specifies which entries from the `files` field in the
  * `package.json` are optional. The key is a package name, and its value is an array of optional entries from the `files` field, for which
  * it is allowed not to match any file. The `options.optionalEntries` object may also contain the `default` key, which is used for all
  * packages that do not have own definition.
- * @param {string} [options.confirmationCallback=null] An callback whose response decides to continue the publishing packages. Synchronous
- * and asynchronous callbacks are supported.
+ * @param {function|null} [options.confirmationCallback=null] An callback whose response decides to continue the publishing packages.
+ * Synchronous and asynchronous callbacks are supported.
  * @param {boolean} [options.requireEntryPoint=false] Whether to verify if packages to publish define an entry point. In other words,
  * whether their `package.json` define the `main` field.
  * @param {Array.<string>} [options.optionalEntryPointPackages=[]] If the entry point validator is enabled (`requireEntryPoint=true`),
  * this array contains a list of packages that will not be checked. In other words, they do not have to define the entry point.
  * @param {string} [options.cwd=process.cwd()] Current working directory from which all paths will be resolved.
  * @param {number} [options.concurrency=4] Number of CPUs that will execute the task.
+ * @param {number} [options.attempts=3] Number of attempts. After reaching 0, it won't be publishing packages again.
  * @returns {Promise}
  */
 export default async function publishPackages( options ) {
@@ -56,33 +62,84 @@ export default async function publishPackages( options ) {
 		requireEntryPoint = false,
 		optionalEntryPointPackages = [],
 		cwd = process.cwd(),
-		concurrency = 4
+		concurrency = 4,
+		attempts = 3
 	} = options;
 
+	const remainingAttempts = attempts - 1;
 	await assertNpmAuthorization( npmOwner );
 
-	const packagePaths = await glob( '*/', {
-		cwd: upath.join( cwd, packagesDirectory ),
-		absolute: true
-	} );
+	const packagePaths = await findPathsToPackages( cwd, packagesDirectory );
 
 	await assertPackages( packagePaths, { requireEntryPoint, optionalEntryPointPackages } );
 	await assertFilesToPublish( packagePaths, optionalEntries );
 	await assertNpmTag( packagePaths, npmTag );
 
 	const shouldPublishPackages = confirmationCallback ? await confirmationCallback() : true;
 
-	if ( shouldPublishPackages ) {
-		await executeInParallel( {
-			cwd,
-			packagesDirectory,
-			listrTask,
-			taskToExecute: publishPackageOnNpmCallback,
-			taskOptions: {
-				npmTag
-			},
-			signal,
-			concurrency
-		} );
+	if ( !shouldPublishPackages ) {
+		return Promise.resolve();
 	}
+
+	await removeAlreadyPublishedPackages( packagePaths );
+
+	await executeInParallel( {
+		cwd,
+		packagesDirectory,
+		listrTask,
+		taskToExecute: publishPackageOnNpmCallback,
+		taskOptions: {
+			npmTag
+		},
+		signal,
+		concurrency
+	} );
+
+	const packagePathsAfterPublishing = await findPathsToPackages( cwd, packagesDirectory );
+
+	// All packages have been published. No need for re-executing.
+	if ( !packagePathsAfterPublishing.length ) {
+		return Promise.resolve();
+	}
+
+	// No more attempts. Abort.
+	if ( remainingAttempts <= 0 ) {
+		throw new Error( 'Some packages could not be published.' );
+	}
+
+	// Let's give an npm a moment for taking a breath...
+	await wait( 1000 );
+
+	// ...and try again.
+	return publishPackages( {
+		packagesDirectory,
+		npmOwner,
+		listrTask,
+		signal,
+		npmTag,
+		optionalEntries,
+		requireEntryPoint,
+		optionalEntryPointPackages,
+		cwd,
+		concurrency,
+		confirmationCallback: null, // Do not ask again if already here.
+		attempts: remainingAttempts
+	} );
+}
+
+async function removeAlreadyPublishedPackages( packagePaths ) {
+	for ( const absolutePackagePath of packagePaths ) {
+		const pkgJson = await fs.readJson( upath.join( absolutePackagePath, 'package.json' ) );
+		const isAvailable = await checkVersionAvailability( pkgJson.version, pkgJson.name );
+
+		if ( !isAvailable ) {
+			await fs.remove( absolutePackagePath );
+		}
+	}
+}
+
+function wait( time ) {
+	return new Promise( resolve => {
+		setTimeout( resolve, time );
+	} );
 }

packages/ckeditor5-dev-release-tools/lib/tasks/updatedependencies.js

@@ -4,8 +4,10 @@
  */
 
 import fs from 'fs-extra';
-import { glob } from 'glob';
 import upath from 'upath';
+import findPathsToPackages from '../utils/findpathstopackages.js';
+
+const { normalizeTrim } = upath;
 
 /**
  * The purpose of this script is to update all eligible dependencies to a version specified in the `options.version`. The following packages
@@ -38,15 +40,15 @@ export default async function updateDependencies( options ) {
 		cwd = process.cwd()
 	} = options;
 
-	const globPatterns = [ 'package.json' ];
-
-	if ( packagesDirectory ) {
-		const packagesDirectoryPattern = upath.join( packagesDirectory, '*', 'package.json' );
-
-		globPatterns.push( packagesDirectoryPattern );
-	}
-
-	const pkgJsonPaths = await getPackageJsonPaths( cwd, globPatterns, packagesDirectoryFilter );
+	const pkgJsonPaths = await findPathsToPackages(
+		cwd,
+		packagesDirectory ? normalizeTrim( packagesDirectory ) : null,
+		{
+			includePackageJson: true,
+			includeCwd: true,
+			packagesDirectoryFilter
+		}
+	);
 
 	for ( const pkgJsonPath of pkgJsonPaths ) {
 		const pkgJson = await fs.readJson( pkgJsonPath );
@@ -78,28 +80,6 @@ function updateVersion( version, callback, dependencies ) {
 	}
 }
 
-/**
- * @param {string} cwd
- * @param {Array.<string>} globPatterns
- * @param {UpdateDependenciesPackagesDirectoryFilter|null} packagesDirectoryFilter
- * @returns {Promise.<Array.<string>>}
- */
-async function getPackageJsonPaths( cwd, globPatterns, packagesDirectoryFilter ) {
-	const globOptions = {
-		cwd,
-		nodir: true,
-		absolute: true
-	};
-
-	const pkgJsonPaths = await glob( globPatterns, globOptions );
-
-	if ( !packagesDirectoryFilter ) {
-		return pkgJsonPaths;
-	}
-
-	return pkgJsonPaths.filter( packagesDirectoryFilter );
-}
-
 /**
  * @callback UpdateVersionCallback
  *