Fix ensoup launcher upgrade mechanism (#11833)

- Closes #11821 by updating the upgrade logic to the new executable name
- Re-enables the long disabled `UpgradeSpec` to make sure this remains tested.
- If the tests were enabled we would have caught the regression in #10535
- The tests have been heavily outdated due to being disabled, many small details changed and had to be amended.
- The tests are still marked as Flaky - they were known to be problematic on CI so their failures will not stop CI for now. But at least they are run and we can see if they succeed or not. Plus when running tests locally they will fail (as all tests marked as Flaky - the failure is only ignored on CI).
- Fixes another issue with an infinite cycle when no upgrade path can be found and adds a test for this case.
- If running a development build, the minimum version check can be ignored, as the check does not really make sense for `0.0.0-dev` build.
- Thus it closes #11831 also.
- Makes sure that `GithubAPI` caches the list of releases as fetching it can take time.

engine/launcher/src/main/scala/org/enso/launcher/InfoLogger.scala

@@ -1,6 +1,5 @@
 package org.enso.launcher
 
-import com.typesafe.scalalogging.Logger
 import org.enso.cli.CLIOutput
 
 /** Handles displaying of user-facing information.
@@ -11,23 +10,12 @@ import org.enso.cli.CLIOutput
   */
 object InfoLogger {
 
-  private val logger = Logger("launcher")
-
   /** Prints an info level message.
     *
-    * If the default logger is set-up to display info-messages, they are send to
-    * the logger, otherwise they are printed to stdout.
-    *
-    * It is important to note that these messages should always be displayed to
-    * the user, so unless run in debug mode, all launcher settings should ensure
-    * that info-level logs are printed to the console output.
+    * Currently, the message is always printed to standard output. But this may be changed by changing this method.
     */
   def info(msg: => String): Unit = {
-    if (logger.underlying.isInfoEnabled) {
-      logger.info(msg)
-    } else {
-      CLIOutput.println(msg)
-    }
+    CLIOutput.println(msg)
   }
 
 }

engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala

@@ -33,6 +33,9 @@ object LauncherRepository {
   private val launcherFallbackProviderHostname =
     "launcherfallback.release.enso.org"
 
+  /** URL to the repo that could be displayed to the user. */
+  def websiteUrl: String = "https://github.com/enso-org/enso"
+
   /** Defines a part of the URL scheme of the fallback mechanism - the name of
     * the directory that holds the releases.
     *

engine/launcher/src/main/scala/org/enso/launcher/releases/launcher/LauncherReleaseProvider.scala

@@ -29,8 +29,7 @@ class LauncherReleaseProvider(releaseProvider: SimpleReleaseProvider)
           .find(_.fileName == LauncherManifest.assetName)
           .toRight(
             ReleaseProviderException(
-              s"${LauncherManifest.assetName} file is missing from release " +
-              s"assets."
+              s"${LauncherManifest.assetName} file is missing from $tag release assets."
             )
           )
           .toTry

engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala

@@ -1,6 +1,6 @@
 package org.enso.launcher.upgrade
 
-import java.nio.file.{Files, Path}
+import java.nio.file.{AccessDeniedException, Files, Path}
 import com.typesafe.scalalogging.Logger
 import org.enso.semver.SemVer
 import org.enso.semver.SemVerOrdering._
@@ -24,7 +24,7 @@ import org.enso.launcher.cli.{
 import org.enso.launcher.releases.launcher.LauncherRelease
 import org.enso.runtimeversionmanager.releases.ReleaseProvider
 import org.enso.launcher.releases.LauncherRepository
-import org.enso.launcher.InfoLogger
+import org.enso.launcher.{Constants, InfoLogger}
 import org.enso.launcher.distribution.DefaultManagers
 import org.enso.runtimeversionmanager.locking.Resources
 import org.slf4j.LoggerFactory
@@ -97,12 +97,45 @@ class LauncherUpgrader(
           }
         }
       }
-      if (release.canPerformUpgradeFromCurrentVersion)
+
+      val canPerformDirectUpgrade: Boolean =
+        if (release.canPerformUpgradeFromCurrentVersion) true
+        else if (CurrentVersion.isDevVersion) {
+          logger.warn(
+            s"Cannot upgrade to version ${release.version} directly, because " +
+            s"it requires at least version " +
+            s"${release.minimumVersionToPerformUpgrade}."
+          )
+          if (globalCLIOptions.autoConfirm) {
+            logger.warn(
+              s"However, the current version (${CurrentVersion.version}) is " +
+              s"a development version, so the minimum version check can be " +
+              s"ignored. Since `auto-confirm` is set, the upgrade will " +
+              s"continue. But please be warned that it may fail due to " +
+              s"incompatibility."
+            )
+            true
+          } else {
+            logger.warn(
+              s"Since the current version (${CurrentVersion.version}) is " +
+              s"a development version, the minimum version check can be " +
+              s"ignored. However, please be warned that the upgrade " +
+              s"may fail due to incompatibility."
+            )
+            CLIOutput.askConfirmation(
+              "Do you want to continue upgrading to this version " +
+              "despite the warning?"
+            )
+          }
+        } else false
+
+      if (canPerformDirectUpgrade)
         performUpgradeTo(release)
       else
         performStepByStepUpgrade(release)
 
       runCleanup()
+      logger.debug("Upgrade completed successfully.")
     }
   }
 
@@ -120,11 +153,13 @@ class LauncherUpgrader(
     val temporaryFiles =
       FileSystem.listDirectory(binRoot).filter(isTemporaryExecutable)
     if (temporaryFiles.nonEmpty && isStartup) {
-      logger.debug("Cleaning temporary files from a previous upgrade.")
+      logger.debug(
+        s"Cleaning ${temporaryFiles.size} temporary files from a previous upgrade."
+      )
     }
     for (file <- temporaryFiles) {
       try {
-        Files.delete(file)
+        tryHardToDelete(file)
         logger.debug(s"Upgrade cleanup: removed `$file`.")
       } catch {
         case NonFatal(e) =>
@@ -133,6 +168,21 @@ class LauncherUpgrader(
     }
   }
 
+  /** On Windows, deleting an executable immediately after it has exited may fail
+    * and the process may need to wait a few millisecond. This method detects
+    * this kind of failure and retries a few times.
+    */
+  private def tryHardToDelete(file: Path, attempts: Int = 30): Unit = {
+    try {
+      Files.delete(file)
+    } catch {
+      case _: AccessDeniedException if attempts > 0 =>
+        logger.trace(s"Failed to delete file `$file`. Retrying.")
+        Thread.sleep(100)
+        tryHardToDelete(file, attempts - 1)
+    }
+  }
+
   /** Continues a multi-step upgrade.
     *
     * Called by [[InternalOpts]] when the upgrade continuation is requested by
@@ -217,28 +267,53 @@ class LauncherUpgrader(
 
   @scala.annotation.tailrec
   private def nextVersionToUpgradeTo(
-    release: LauncherRelease,
+    currentTargetRelease: LauncherRelease,
     availableVersions: Seq[SemVer]
   ): LauncherRelease = {
-    val recentEnoughVersions =
-      availableVersions.filter(
-        _.isGreaterThanOrEqual(release.minimumVersionToPerformUpgrade)
+    assert(
+      currentTargetRelease.minimumVersionToPerformUpgrade.isGreaterThan(
+        CurrentVersion.version
       )
+    )
+
+    // We look at older versions that are satisfying the minimum version
+    // required to upgrade to currentTargetRelease.
+    val recentEnoughVersions =
+      availableVersions.filter { possibleVersion =>
+        val canUpgradeToTarget = possibleVersion.isGreaterThanOrEqual(
+          currentTargetRelease.minimumVersionToPerformUpgrade
+        )
+        val isEarlierThanTarget =
+          possibleVersion.isLessThan(currentTargetRelease.version)
+        canUpgradeToTarget && isEarlierThanTarget
+      }
+
+    // We take the oldest of these, hoping that it will yield the shortest
+    // upgrade path (perhaps it will be possible to upgrade directly from
+    // current version)
     val minimumValidVersion = recentEnoughVersions.sorted.headOption.getOrElse {
       throw UpgradeError(
-        s"Upgrade failed: To continue upgrade, a version at least " +
-        s"${release.minimumVersionToPerformUpgrade} is required, but no " +
-        s"valid version satisfying this requirement could be found."
+        s"Upgrade failed: To continue upgrade, at least version " +
+        s"${currentTargetRelease.minimumVersionToPerformUpgrade} is required, " +
+        s"but no upgrade path has been found from the current version " +
+        s"${CurrentVersion.version}. " +
+        s"Please manually download a newer release from " +
+        s"${LauncherRepository.websiteUrl}"
       )
     }
-    val nextRelease = releaseProvider.fetchRelease(minimumValidVersion).get
+
+    val newTargetRelease = releaseProvider.fetchRelease(minimumValidVersion).get
+    assert(newTargetRelease.version != currentTargetRelease.version)
     logger.debug(
-      s"To upgrade to ${release.version}, " +
-      s"the launcher will have to upgrade to ${nextRelease.version} first."
+      s"To upgrade to ${currentTargetRelease.version}, " +
+      s"the launcher will have to upgrade to ${newTargetRelease.version} first."
     )
-    if (nextRelease.canPerformUpgradeFromCurrentVersion)
-      nextRelease
-    else nextVersionToUpgradeTo(nextRelease, availableVersions)
+
+    // If the current version cannot upgrade directly to the new target version,
+    // we continue the search looking for an even earlier version that we could
+    // upgrade to.
+    if (newTargetRelease.canPerformUpgradeFromCurrentVersion) newTargetRelease
+    else nextVersionToUpgradeTo(newTargetRelease, availableVersions)
   }
 
   /** Extracts just the launcher executable from the archive.
@@ -331,7 +406,7 @@ class LauncherUpgrader(
 
       val temporaryExecutable = temporaryExecutablePath("new")
       FileSystem.copyFile(
-        extractedRoot / "bin" / OS.executableName("enso"),
+        extractedRoot / "bin" / OS.executableName(Constants.name),
         temporaryExecutable
       )
 

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-0.9999.0-marked-broken/launcher-manifest.yaml → engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.1/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 1.0.0
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-0.0.1/launcher-manifest.yaml → engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.2/launcher-manifest.yaml

@@ -1,4 +1,4 @@
-minimum-version-for-upgrade: 0.0.0
+minimum-version-for-upgrade: 1.0.1
 files-to-copy:
   - README.md
 directories-to-copy:

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-0.0.2/launcher-manifest.yaml → engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.3/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.1
+minimum-version-for-upgrade: 1.0.2
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-0.0.3/launcher-manifest.yaml → engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-1.0.4/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 1.0.2
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-0.0.4/launcher-manifest.yaml → engine/launcher/src/test/resources/org/enso/launcher/components/fake-releases/launcher/enso-99.9999.0/launcher-manifest.yaml

@@ -1,3 +1,3 @@
-minimum-version-for-upgrade: 0.0.2
+minimum-version-for-upgrade: 0.0.0
 files-to-copy: []
 directories-to-copy: []

engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala

@@ -58,7 +58,7 @@ trait NativeTest
     args: Seq[String],
     extraEnv: Map[String, String]      = Map.empty,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -89,7 +89,7 @@ trait NativeTest
     args: Seq[String],
     extraEnv: Map[String, String]      = Map.empty,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -146,7 +146,7 @@ trait NativeTest
     args: Seq[String],
     pathOverride: String,
     extraJVMProps: Map[String, String] = Map.empty,
-    timeoutSeconds: Long               = 15
+    timeoutSeconds: Long               = defaultTimeoutSeconds
   ): RunResult = {
     runCommand(
       Seq(baseLauncherLocation.toAbsolutePath.toString) ++ args,
@@ -155,6 +155,8 @@ trait NativeTest
       timeoutSeconds = timeoutSeconds
     )
   }
+
+  private val defaultTimeoutSeconds: Long = 30
 }
 
 object NativeTest {