Make the refreshVersions task fully support configuration cache

Related to #672

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/RefreshVersionsCorePlugin.kt

@@ -5,7 +5,7 @@ import de.fayard.refreshVersions.core.extensions.gradle.isRootProject
 import de.fayard.refreshVersions.core.internal.InternalRefreshVersionsApi
 import de.fayard.refreshVersions.core.internal.OutputFile
 import de.fayard.refreshVersions.core.internal.RefreshVersionsConfigHolder
-import de.fayard.refreshVersions.core.internal.skipConfigurationCache
+import de.fayard.refreshVersions.core.internal.VersionsCatalogs
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.kotlin.dsl.register
@@ -18,15 +18,31 @@ open class RefreshVersionsCorePlugin : Plugin<Project> {
         check(project.isRootProject) { "ERROR: de.fayard.refreshVersions.core should not be applied manually" }
         if (project.isBuildSrc.not()) {
             OutputFile.init(project)
-            // In the case where this runs in includedBuilds, the task configuration lambda may (will) run
-            // after RefreshVersionsConfigHolder content is cleared (via its ClearStaticStateBuildService),
-            // so we get the value before.
             val versionsFileName = RefreshVersionsConfigHolder.versionsPropertiesFile.name
 
+            val shouldUpdateVersionCatalogs = VersionsCatalogs.isSupported() && FeatureFlag.VERSIONS_CATALOG.isEnabled
             project.tasks.register<RefreshVersionsTask>(name = "refreshVersions") {
                 group = "refreshVersions"
                 description = "Search for new dependencies versions and update $versionsFileName"
-                skipConfigurationCache()
+                if (shouldUpdateVersionCatalogs) {
+                    this.defaultVersionCatalog = VersionsCatalogs.getDefault(project)
+                    this.defaultVersionCatalogFile = project.file(VersionsCatalogs.LIBS_VERSIONS_TOML)
+                }
+                rootProjectSettingsFile = project.file("settings.gradle.kts").let { kotlinDslSettings ->
+                    if (kotlinDslSettings.exists()) kotlinDslSettings else {
+                        project.file("settings.gradle").also {
+                            check(it.exists())
+                        }
+                    }
+                }
+                buildSrcSettingsFile = project.file("buildSrc/settings.gradle.kts").let { kotlinDslSettings ->
+                    if (kotlinDslSettings.exists()) kotlinDslSettings else {
+                        project.file("buildSrc/settings.gradle").takeIf {
+                            it.exists()
+                        }
+                    }
+                }
+                RefreshVersionsConfigHolder.dependenciesTracker.recordBuildscriptAndRegularDependencies(project)
             }
 
             project.tasks.register<RefreshVersionsCleanupTask>(name = "refreshVersionsCleanup") {

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/RefreshVersionsTask.kt

@@ -1,8 +1,14 @@
 package de.fayard.refreshVersions.core
 
-import de.fayard.refreshVersions.core.internal.*
+import de.fayard.refreshVersions.core.internal.OutputFile
+import de.fayard.refreshVersions.core.internal.PluginDependencyCompat
+import de.fayard.refreshVersions.core.internal.RefreshVersionsConfigHolder
 import de.fayard.refreshVersions.core.internal.RefreshVersionsConfigHolder.settings
-import de.fayard.refreshVersions.core.internal.VersionsCatalogs.LIBS_VERSIONS_TOML
+import de.fayard.refreshVersions.core.internal.SettingsPluginsUpdater
+import de.fayard.refreshVersions.core.internal.VersionsCatalogUpdater
+import de.fayard.refreshVersions.core.internal.VersionsCatalogs
+import de.fayard.refreshVersions.core.internal.configureLintIfRunningOnAnAndroidProject
+import de.fayard.refreshVersions.core.internal.lookupVersionCandidates
 import de.fayard.refreshVersions.core.internal.problems.log
 import de.fayard.refreshVersions.core.internal.versions.VersionsPropertiesModel
 import de.fayard.refreshVersions.core.internal.versions.writeWithNewVersions
@@ -11,11 +17,14 @@ import kotlinx.coroutines.runBlocking
 import org.gradle.api.DefaultTask
 import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.MinimalExternalModuleDependency
+import org.gradle.api.artifacts.VersionCatalog
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
 import org.gradle.util.GradleVersion
+import java.io.File
 
 /**
  *
@@ -29,7 +38,6 @@ import org.gradle.util.GradleVersion
  */
 open class RefreshVersionsTask : DefaultTask() {
 
-
     @Input @Optional
     @Option(option = "enable", description = "Enable a feature flag")
     var enableFlag: FeatureFlag? = null
@@ -46,6 +54,20 @@ open class RefreshVersionsTask : DefaultTask() {
             if (value != null) FeatureFlag.userSettings.put(value, false)
         }
 
+    @get:Input
+    @get:Optional
+    internal var defaultVersionCatalog: VersionCatalog? = null
+
+    @get:InputFile
+    internal lateinit var defaultVersionCatalogFile: File
+
+    @get:InputFile
+    internal lateinit var rootProjectSettingsFile: File
+
+    @get:InputFile
+    @get:Optional
+    internal var buildSrcSettingsFile: File? = null
+
     @TaskAction
     fun taskActionRefreshVersions() {
         OutputFile.checkWhichFilesExist()
@@ -62,7 +84,7 @@ open class RefreshVersionsTask : DefaultTask() {
         val versionsCatalogLibraries: Set<MinimalExternalModuleDependency>
         val versionsCatalogPlugins: Set<PluginDependencyCompat>
         if (shouldUpdateVersionCatalogs) {
-            val versionCatalog = VersionsCatalogs.getDefault(project)
+            val versionCatalog = defaultVersionCatalog
             versionsCatalogLibraries = VersionsCatalogs.libraries(versionCatalog)
             versionsCatalogPlugins = VersionsCatalogs.plugins(versionCatalog)
         } else {
@@ -80,16 +102,17 @@ open class RefreshVersionsTask : DefaultTask() {
             }) { httpClient ->
                 lookupVersionCandidates(
                     httpClient = httpClient,
-                    project = project,
+                    dependenciesTracker = RefreshVersionsConfigHolder.dependenciesTracker,
                     versionMap = RefreshVersionsConfigHolder.readVersionsMap(),
                     versionKeyReader = RefreshVersionsConfigHolder.versionKeyReader,
                     versionsCatalogLibraries = versionsCatalogLibraries,
                     versionsCatalogPlugins = versionsCatalogPlugins
                 )
             }
-            VersionsPropertiesModel.writeWithNewVersions(result.dependenciesUpdatesForVersionsProperties)
+            val versionPropertiesUpdated = VersionsPropertiesModel.writeWithNewVersions(result.dependenciesUpdatesForVersionsProperties)
             SettingsPluginsUpdater.updateGradleSettingsWithAvailablePluginsUpdates(
-                rootProject = project,
+                rootProjectSettingsFile = rootProjectSettingsFile,
+                buildSrcSettingsFile = buildSrcSettingsFile,
                 settingsPluginsUpdates = result.settingsPluginsUpdates,
                 buildSrcSettingsPluginsUpdates = result.buildSrcSettingsPluginsUpdates
             )
@@ -104,7 +127,7 @@ open class RefreshVersionsTask : DefaultTask() {
             if (versionPropertiesUpdated) OutputFile.VERSIONS_PROPERTIES.logFileWasModified()
 
             if (shouldUpdateVersionCatalogs) {
-                val libsToml = project.file(LIBS_VERSIONS_TOML)
+                val libsToml = defaultVersionCatalogFile
                 if (libsToml.canRead()) {
                     val updated = VersionsCatalogUpdater(
                         file = libsToml,

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/extensions/gradle/Dependency.kt

@@ -1,12 +1,14 @@
 package de.fayard.refreshVersions.core.extensions.gradle
 
 import de.fayard.refreshVersions.core.ModuleId
+import de.fayard.refreshVersions.core.internal.ConfigurationLessDependency
 import de.fayard.refreshVersions.core.internal.InternalRefreshVersionsApi
 import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ExternalDependency
 
 @InternalRefreshVersionsApi
 fun Dependency.moduleId(): ModuleId? = when {
+    this is ConfigurationLessDependency -> if (isNpm) npmModuleId() else mavenModuleId()
     this is ExternalDependency -> mavenModuleId()
     this::class.simpleName == "NpmDependency" -> npmModuleId()
     else -> null

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/ConfigurationLessDependency.kt

@@ -2,13 +2,16 @@ package de.fayard.refreshVersions.core.internal
 
 import de.fayard.refreshVersions.core.ModuleId
 import org.gradle.api.artifacts.Dependency
-import org.gradle.api.internal.artifacts.dependencies.AbstractDependency
+import org.gradle.api.artifacts.ExternalDependency
+import org.gradle.api.artifacts.VersionConstraint
 
-internal class ConfigurationLessDependency(
-    private val group: String,
+internal data class ConfigurationLessDependency(
+    private val group: String?,
     private val name: String,
-    private val version: String?
-) : AbstractDependency() {
+    private val version: String?,
+    val versionConstraint: VersionConstraint? = null,
+    val isNpm: Boolean = false
+) : Dependency {
 
     constructor(
         moduleId: ModuleId.Maven,
@@ -19,6 +22,16 @@ internal class ConfigurationLessDependency(
         version = version
     )
 
+    constructor(
+        otherDependency: Dependency
+    ) : this(
+        group = otherDependency.group,
+        name = otherDependency.name,
+        version = otherDependency.version,
+        versionConstraint = if (otherDependency is ExternalDependency) otherDependency.versionConstraint else null,
+        isNpm = otherDependency::class.simpleName == "NpmDependency"
+    )
+
     companion object {
         operator fun invoke(dependencyNotation: String): ConfigurationLessDependency {
             val beforeFirstColon = dependencyNotation.substringBefore(':')
@@ -37,13 +50,14 @@ internal class ConfigurationLessDependency(
     override fun getName() = name
     override fun getVersion() = version
 
-    override fun contentEquals(dependency: Dependency): Boolean = throw UnsupportedOperationException()
+    override fun contentEquals(dependency: Dependency): Boolean = when (dependency) {
+        is ConfigurationLessDependency -> dependency == this
+        else -> dependency.contentEquals(this)
+    }
 
-    override fun copy(): Dependency = ConfigurationLessDependency(
-        group = group,
-        name = name,
-        version = version
-    )
+    override fun copy(): Dependency = copy(group = group)
+    override fun getReason(): String? = null
+    override fun because(reason: String?) = throw IllegalAccessException("Not editable")
 
     override fun toString(): String = if (version == null) "$group:$name" else "$group:$name:$version"
 }

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/DependencyVersionsFetcher.Companion.kt

@@ -3,6 +3,7 @@ package de.fayard.refreshVersions.core.internal
 import de.fayard.refreshVersions.core.DependencyVersionsFetcher
 import de.fayard.refreshVersions.core.ModuleId
 import de.fayard.refreshVersions.core.extensions.gradle.passwordCredentials
+import de.fayard.refreshVersions.core.internal.dependencies.Repo
 import de.fayard.refreshVersions.core.internal.npm.NpmDependencyVersionsFetcherHttp
 import okhttp3.Credentials
 import okhttp3.OkHttpClient
@@ -38,7 +39,41 @@ internal fun DependencyVersionsFetcher.Companion.forMaven(
         // https://docs.gradle.org/current/userguide/declaring_repositories.html#sec:supported_transport_protocols
         // We should trigger a warning that it's not supported yet, with link to relevant issue,
         // and we should report any transport protocol not known to be supported by Gradle (but not crash
-        // in case a future Gradle version adds more of them and we're not updated yet to support it).
+        // in case a future Gradle version adds more of them, and we're not updated yet to support it).
+    }
+}
+
+internal fun DependencyVersionsFetcher.Companion.forMaven(
+    httpClient: OkHttpClient,
+    moduleId: ModuleId.Maven?,
+    repository: Repo.Maven // TODO: Support Ivy repositories
+): DependencyVersionsFetcher? {
+    moduleId ?: return null
+    return when (repository.url.scheme) {
+        "https", "http" -> MavenDependencyVersionsFetcherHttp(
+            httpClient = httpClient,
+            moduleId = moduleId,
+            repoUrl = repository.url.toString().let { if (it.endsWith('/')) it else "$it/" },
+            repoAuthorization = repository.passwordCredentials?.let { credentials ->
+                Credentials.basic(
+                    username = credentials.username,
+                    password = credentials.password
+                )
+            }
+        )
+        "file" -> MavenDependencyVersionsFetcherFile(
+            moduleId = moduleId,
+            repoUrl = repository.url.toString().let { if (it.endsWith('/')) it else "$it/" }
+        )
+        "gcs" -> MavenDependencyVersionsFetcherGoogleCloudStorage(
+            moduleId = moduleId,
+            repoUrl = repository.url.toString()
+        )
+        else -> null //TODO: Support more transport protocols. Here's what Gradle supports:
+        // https://docs.gradle.org/current/userguide/declaring_repositories.html#sec:supported_transport_protocols
+        // We should trigger a warning that it's not supported yet, with link to relevant issue,
+        // and we should report any transport protocol not known to be supported by Gradle (but not crash
+        // in case a future Gradle version adds more of them, and we're not updated yet to support it).
     }
 }
 

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/DependencyWithVersionCandidates.kt

@@ -4,7 +4,7 @@ import de.fayard.refreshVersions.core.DependencyVersionsFetcher
 import de.fayard.refreshVersions.core.ModuleId
 import de.fayard.refreshVersions.core.Version
 
-internal class DependencyWithVersionCandidates(
+internal data class DependencyWithVersionCandidates(
     val moduleId: ModuleId,
     val currentVersion: String, // TODO: Ensure VersionsCatalogUpdater can have the data it needs, and remove this.
     val versionsCandidates: (currentVersion: Version) -> List<Version>,

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/NewRefreshVersionsImpl.kt → plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/LookupVersionCandidates.kt

@@ -7,8 +7,8 @@ import de.fayard.refreshVersions.core.ModuleId
 import de.fayard.refreshVersions.core.Version
 import de.fayard.refreshVersions.core.extensions.gradle.*
 import de.fayard.refreshVersions.core.extensions.gradle.hasDynamicVersion
-import de.fayard.refreshVersions.core.extensions.gradle.isRootProject
 import de.fayard.refreshVersions.core.extensions.gradle.npmModuleId
+import de.fayard.refreshVersions.core.internal.dependencies.DependenciesTracker
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
@@ -25,31 +25,22 @@ import org.gradle.util.GradleVersion
 
 internal suspend fun lookupVersionCandidates(
     httpClient: OkHttpClient,
-    project: Project,
+    dependenciesTracker: DependenciesTracker,
     versionMap: Map<String, String>,
     versionKeyReader: ArtifactVersionKeyReader,
     versionsCatalogLibraries: Set<MinimalExternalModuleDependency>,
     versionsCatalogPlugins: Set<PluginDependencyCompat>
 ): VersionCandidatesLookupResult {
 
-    require(project.isRootProject)
-
-    val projects = RefreshVersionsConfigHolder.allProjects(project)
-
     val dependenciesFromVersionFor = UsedVersionForTracker.read()
 
     return lookupVersionCandidates(
         dependencyVersionsFetchers = { dependencyFilter ->
-            projects.flatMap {
-                it.getDependencyVersionFetchers(httpClient = httpClient, dependencyFilter = dependencyFilter).toList()
-            }.plus(
-                UsedPluginsTracker.read().getDependencyVersionsFetchers(httpClient) //TODO: Is this needed?
-                //TODO: If so, don't we miss passing dependencies through the dependencyFilter?
-            ).plus(
+            dependenciesTracker.dependencyVersionsFetchers(httpClient, dependencyFilter).plus(
                 dependenciesFromVersionFor.asSequence().onEach { (dependency, _) ->
                     check(dependencyFilter(dependency)) // Needed because dependencyFilter also tracks dependencies usage.
                 }.getDependencyVersionsFetchers(httpClient)
-            ).toSet()
+            )
         },
         lookupAvailableGradleVersions = {
             if (GRADLE_UPDATES.isEnabled) lookupAvailableGradleVersions(httpClient) else emptyList()
@@ -232,23 +223,6 @@ internal fun Settings.getDependencyVersionFetchers(
     dependencyFilter = dependencyFilter
 )
 
-private fun Project.getDependencyVersionFetchers(
-    httpClient: OkHttpClient,
-    dependencyFilter: (Dependency) -> Boolean
-): Sequence<DependencyVersionsFetcher> = getDependencyVersionFetchers(
-    httpClient = httpClient,
-    configurations = buildscript.configurations,
-    repositories = buildscript.repositories.withPluginsRepos(),
-    dependencyFilter = dependencyFilter
-).plus(
-    getDependencyVersionFetchers(
-        httpClient = httpClient,
-        configurations = configurations,
-        repositories = repositories,
-        dependencyFilter = dependencyFilter
-    )
-)
-
 private fun Sequence<Pair<Dependency, List<ArtifactRepository>>>.getDependencyVersionsFetchers(
     httpClient: OkHttpClient
 ): Sequence<DependencyVersionsFetcher> {

plugins/core/src/main/kotlin/de/fayard/refreshVersions/core/internal/RefreshVersionsConfigHolder.kt

@@ -3,7 +3,7 @@ package de.fayard.refreshVersions.core.internal
 import de.fayard.refreshVersions.core.DependencySelection
 import de.fayard.refreshVersions.core.ModuleId
 import de.fayard.refreshVersions.core.extensions.gradle.isBuildSrc
-import de.fayard.refreshVersions.core.extensions.gradle.isRootProject
+import de.fayard.refreshVersions.core.internal.dependencies.DependenciesTracker
 import de.fayard.refreshVersions.core.internal.versions.VersionsPropertiesModel
 import de.fayard.refreshVersions.core.internal.versions.VersionsPropertiesModel.Section.VersionEntry
 import de.fayard.refreshVersions.core.internal.versions.readFromFile
@@ -32,7 +32,7 @@ object RefreshVersionsConfigHolder {
     var versionsPropertiesFile: File by resettableDelegates.LateInit()
         private set
 
-    val buildSrc: Project? get() = buildSrcSettings?.gradle?.rootProject
+    internal val dependenciesTracker: DependenciesTracker by resettableDelegates.Lazy { DependenciesTracker() }
 
     /**
      * Not initialized when the IDE syncs the buildSrc of the project alone (without the host project)
@@ -56,12 +56,6 @@ object RefreshVersionsConfigHolder {
         }
     }
 
-    fun allProjects(project: Project): Sequence<Project> {
-        require(project.isRootProject)
-        val buildSrcProjects = buildSrc?.allprojects?.asSequence() ?: emptySequence()
-        return buildSrcProjects + project.allprojects.asSequence()
-    }
-
     internal val resultMode: VersionCandidatesResultMode = VersionCandidatesResultMode(
         filterMode = VersionCandidatesResultMode.FilterMode.AllIntermediateVersions,
         sortingMode = VersionCandidatesResultMode.SortingMode.ByVersion
@@ -123,6 +117,7 @@ object RefreshVersionsConfigHolder {
         // requested?
         // Might be worth opening an issue on https://youtrack.jetbrains.com to find out.
         if (versionKeyReaderDelegate.isInitialized) {
+            dependenciesTracker.recordBuildscriptAndRegularDependencies(settings)
             persistInitData(settings)
         } else {
             runCatching {