Skip to content

Commit

Permalink
Fix IDEA-347713 Package Search Plugin high CPU use (#112)
Browse files Browse the repository at this point in the history 
This update modifies the caching system and retrieval of modules in Gradle projects. It revamps the caching of declared dependencies by associating them with their SHA hash of the respective build file, and storing them in the local caches.

Co-authored-by: Lamberto Basti <[email protected]>
  • Loading branch information
@lamba92
lamba92 and Lamberto Basti authored Feb 28, 2024
1 parent c917125 commit acffb8f
Show file tree
Hide file tree
Showing 11 changed files with 299 additions and 160 deletions.
3 changes: 2 additions & 1 deletion ...n/com/jetbrains/packagesearch/plugin/core/nitrite/coroutines/CoroutineObjectRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.jetbrains.packagesearch.plugin.core.nitrite.coroutines
import com.jetbrains.packagesearch.plugin.core.nitrite.DocumentPathBuilder
import com.jetbrains.packagesearch.plugin.core.nitrite.asKotlin
import com.jetbrains.packagesearch.plugin.core.nitrite.serialization.NitriteDocumentFormat
import java.io.Closeable
import kotlin.reflect.KProperty
import kotlin.reflect.KType
import kotlinx.coroutines.CoroutineDispatcher
Expand Down Expand Up @@ -35,7 +36,7 @@ class CoroutineObjectRepository<T : Any> @InternalAPI constructor(
val type: KType,
private val documentFormat: NitriteDocumentFormat,
override val dispatcher: CoroutineDispatcher = Dispatchers.IO,
) : CoroutineWrapper() {
) : CoroutineWrapper(), Closeable by synchronous {

data class Change<T>(val changeType: ChangeType, val changedItems: Flow<T>)
data class Item<T>(val changeTimestamp: Instant, val changeType: ChangeType, val item: T)
Expand Down
2 changes: 1 addition & 1 deletion plugin/core/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ tasks {
pluginId = pkgsPluginId
outputDir = generatedDir
packageName = "com.jetbrains.packagesearch.plugin.core"
databaseVersion = 1
databaseVersion = 2
}
sourcesJar {
dependsOn(generatePluginDataSources)
Expand Down
5 changes: 1 addition & 4 deletions plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/extensions/Contexts.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@ interface PackageSearchApiPackagesContext {
}

interface PackageSearchModuleBuilderContext :
ProjectContext, PackageSearchKnownRepositoriesContext, PackageSearchApiPackagesContext {
val projectCaches: CoroutineNitrite
val applicationCaches: CoroutineNitrite
}
ProjectContext, PackageSearchKnownRepositoriesContext, PackageSearchApiPackagesContext

interface ProjectContext {
val project: Project
Expand Down
4 changes: 3 additions & 1 deletion ...le/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ class GradleModuleProvider : AbstractGradleModuleProvider() {
val configurationNames = model.configurations
.filter { it.canBeDeclared }
.map { it.name }
val declaredDependencies = module.getDeclaredDependencies()
val declaredDependencies = model.buildFilePath
?.let { module.getDeclaredDependencies(it) }
?: emptyList()
val packageSearchGradleModule = PackageSearchGradleModule(
name = model.projectName,
identity = PackageSearchModule.Identity(
Expand Down
190 changes: 188 additions & 2 deletions plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,44 @@
package com.jetbrains.packagesearch.plugin.gradle

import com.intellij.openapi.Disposable
import com.intellij.openapi.application.readAction
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.Service.Level
import com.intellij.openapi.components.service
import com.intellij.openapi.module.Module
import com.intellij.openapi.project.Project
import com.intellij.packageSearch.mppDependencyUpdater.MppDependency
import com.intellij.packageSearch.mppDependencyUpdater.MppDependencyModifier
import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel
import com.jetbrains.packagesearch.plugin.core.data.EditModuleContext
import com.jetbrains.packagesearch.plugin.core.data.IconProvider
import com.jetbrains.packagesearch.plugin.core.data.PackageSearchDeclaredPackage
import com.jetbrains.packagesearch.plugin.core.data.PackageSearchModuleVariant
import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext
import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
import com.jetbrains.packagesearch.plugin.core.nitrite.insert
import com.jetbrains.packagesearch.plugin.core.utils.NioPathSerializer
import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService
import com.jetbrains.packagesearch.plugin.core.utils.icon
import com.jetbrains.packagesearch.plugin.core.utils.parseAttributesFromRawStrings
import com.jetbrains.packagesearch.plugin.gradle.utils.GradleDependencyModelCacheEntry
import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies
import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel
import java.nio.file.Path
import korlibs.crypto.SHA256
import kotlin.contracts.contract
import kotlin.io.path.absolutePathString
import kotlin.io.path.isRegularFile
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.singleOrNull
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import org.jetbrains.packagesearch.api.v3.ApiMavenPackage
import org.jetbrains.packagesearch.api.v3.ApiPackage
import org.jetbrains.packagesearch.api.v3.search.buildPackageTypes
import org.jetbrains.packagesearch.api.v3.search.kotlinMultiplatform
import org.jetbrains.packagesearch.packageversionutils.normalization.NormalizedVersion

fun Set<MppCompilationInfoModel.Compilation>.buildAttributes(): List<PackageSearchModuleVariant.Attribute> {
val rawStrings = this@buildAttributes.mapNotNull {
Expand All @@ -25,7 +57,6 @@ fun Set<MppCompilationInfoModel.Compilation>.buildAttributes(): List<PackageSear
return rawStrings.parseAttributesFromRawStrings()
}


internal fun PackageSearchModuleVariant.Attribute.flatAttributesNames(): Set<String> {
return when (this) {
is PackageSearchModuleVariant.Attribute.StringAttribute -> setOf(value)
Expand Down Expand Up @@ -107,4 +138,159 @@ fun PackageSearchKotlinMultiplatformDeclaredDependency.Maven.toMPPDependency() =
artifactId = artifactId,
version = declaredVersion?.versionName,
configuration = configuration,
)
)


context(PackageSearchModuleBuilderContext)
suspend fun Module.getKMPVariants(
compilationModel: Map<String, Set<MppCompilationInfoModel.Compilation>>,
buildFilePath: Path?,
availableScopes: List<String>,
): List<PackageSearchKotlinMultiplatformVariant> = coroutineScope {
if (buildFilePath == null) return@coroutineScope emptyList()

val dependenciesBlockVariant = async {
val declaredDependencies = getDeclaredDependencies(buildFilePath)
PackageSearchKotlinMultiplatformVariant.DependenciesBlock(
declaredDependencies = declaredDependencies.asKmpVariantDependencies(),
compatiblePackageTypes = buildPackageTypes {
mavenPackages()
gradlePackages {
isRootPublication = true
}
},
availableScopes = availableScopes,
defaultScope = "implementation".takeIf { it in availableScopes }
?: declaredDependencies.map { it.configuration }
.groupBy { it }
.mapValues { it.value.count() }
.entries
.maxByOrNull { it.value }
?.key
?: availableScopes.first()
)
}

val rawDeclaredSourceSetDependencies = getDependenciesBySourceSet(buildFilePath)

val packageIds = rawDeclaredSourceSetDependencies
.values
.asSequence()
.flatten()
.distinct()
.map { it.packageId }

val dependencyInfo = getPackageInfoByIdHashes(packageIds.map { ApiPackage.hashPackageId(it) }.toSet())

val declaredSourceSetDependencies =
rawDeclaredSourceSetDependencies
.mapValues { (sourceSetName, dependencies) ->
dependencies.map { artifactModel ->
PackageSearchKotlinMultiplatformDeclaredDependency.Maven(
id = artifactModel.packageId,
declaredVersion = artifactModel.version?.let { NormalizedVersion.fromStringOrNull(it) },
remoteInfo = dependencyInfo[artifactModel.packageId] as? ApiMavenPackage,
declarationIndexes = artifactModel.indexes,
groupId = artifactModel.groupId,
artifactId = artifactModel.artifactId,
variantName = sourceSetName,
configuration = artifactModel.configuration,
icon = dependencyInfo[artifactModel.packageId]?.icon
?: IconProvider.Icons.GRADLE
)
}
}
val sourceSetVariants = compilationModel
.mapKeys { it.key }
.map { (sourceSetName, compilationTargets) ->
PackageSearchKotlinMultiplatformVariant.SourceSet(
name = sourceSetName,
declaredDependencies = declaredSourceSetDependencies[sourceSetName] ?: emptyList(),
attributes = compilationTargets.buildAttributes(),
compatiblePackageTypes = buildPackageTypes {
gradlePackages {
kotlinMultiplatform {
compilationTargets.forEach { compilationTarget ->
when {
compilationTarget is MppCompilationInfoModel.Js -> when (compilationTarget.compiler) {
MppCompilationInfoModel.Js.Compiler.IR -> jsIr()
MppCompilationInfoModel.Js.Compiler.LEGACY -> jsLegacy()
}

compilationTarget is MppCompilationInfoModel.Native -> native(compilationTarget.target)
compilationTarget == MppCompilationInfoModel.Wasm -> wasm()
}
}
when {
MppCompilationInfoModel.Android in compilationTargets -> android()
MppCompilationInfoModel.Jvm in compilationTargets -> jvm()
}
}
}
},
compilerTargets = compilationTargets
)
}

sourceSetVariants + dependenciesBlockVariant.await()
}

@Serializable
data class GradleKMPDependencyModelCacheEntry(
@SerialName("_id") val id: Long? = null,
val buildFile: String,
val buildFileSha: String,
val dependencies: Map<String, List<GradleDependencyModel>>,
)

@Service(Level.PROJECT)
class GradleKMPCacheService(project: Project) : Disposable {
val kmpDependencyRepository =
project.PackageSearchProjectCachesService.getRepository<GradleKMPDependencyModelCacheEntry>("gradle-kmp-dependencies")

override fun dispose() = kmpDependencyRepository.close()
}

context(PackageSearchModuleBuilderContext)
private suspend fun Module.getDependenciesBySourceSet(buildFilePath: Path): Map<String, List<GradleDependencyModel>> {
if (!buildFilePath.isRegularFile()) return emptyMap()

val buildFileHash = SHA256()
.update(buildFilePath.toFile().readBytes())
.digest()
.hex

val entry = project.service<GradleKMPCacheService>()
.kmpDependencyRepository
.find(
filter = NitriteFilters.Object.eq(
path = GradleDependencyModelCacheEntry::buildFile,
value = buildFilePath.absolutePathString()
)
).singleOrNull()

if (entry?.buildFileSha == buildFileHash) return entry.dependencies

val filteredDependenciesBySourceSet =
MppDependencyModifier.dependenciesBySourceSet(this)
?.filterNotNullValues()
?.mapValues { readAction { it.value.artifacts().map { it.toGradleDependencyModel() } }.distinct() }
?: emptyMap()

project.service<GradleKMPCacheService>()
.kmpDependencyRepository
.update(
filter = NitriteFilters.Object.eq(
path = GradleDependencyModelCacheEntry::buildFile,
value = buildFilePath.absolutePathString()
),
update = GradleKMPDependencyModelCacheEntry(
buildFile = buildFilePath.absolutePathString(),
buildFileSha = buildFileHash,
dependencies = filteredDependenciesBySourceSet
),
upsert = true
)

return filteredDependenciesBySourceSet
}
97 changes: 2 additions & 95 deletions ...ain/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.core.utils.icon
import com.jetbrains.packagesearch.plugin.core.utils.toDirectory
import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies
import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel
import java.nio.file.Path
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.FlowCollector
Expand All @@ -41,6 +42,7 @@ class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() {
.collect { compilationModel ->
val variants = module.getKMPVariants(
compilationModel = compilationModel,
buildFilePath = model.buildFilePath,
availableScopes = model.configurations
.filter { it.canBeDeclared }
.map { it.name }
Expand Down Expand Up @@ -68,100 +70,5 @@ class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() {

}

context(PackageSearchModuleBuilderContext)
suspend fun Module.getKMPVariants(
compilationModel: Map<String, Set<MppCompilationInfoModel.Compilation>>,
availableScopes: List<String>,
): List<PackageSearchKotlinMultiplatformVariant> = coroutineScope {
val dependenciesBlockVariant = async {
val declaredDependencies = getDeclaredDependencies()
PackageSearchKotlinMultiplatformVariant.DependenciesBlock(
declaredDependencies = declaredDependencies.asKmpVariantDependencies(),
compatiblePackageTypes = buildPackageTypes {
mavenPackages()
gradlePackages {
isRootPublication = true
}
},
availableScopes = availableScopes,
defaultScope = "implementation".takeIf { it in availableScopes }
?: declaredDependencies.map { it.configuration }
.groupBy { it }
.mapValues { it.value.count() }
.entries
.maxByOrNull { it.value }
?.key
?: availableScopes.first()
)
}

val rawDeclaredSourceSetDependencies = MppDependencyModifier
.dependenciesBySourceSet(this@getKMPVariants)
?.filterNotNullValues()
?.mapValues { readAction { it.value.artifacts().map { it.toGradleDependencyModel() } }.distinct() }
?: emptyMap()

val packageIds = rawDeclaredSourceSetDependencies
.values
.asSequence()
.flatten()
.distinct()
.map { it.packageId }

val dependencyInfo = getPackageInfoByIdHashes(packageIds.map { ApiPackage.hashPackageId(it) }.toSet())

val declaredSourceSetDependencies =
rawDeclaredSourceSetDependencies
.mapValues { (sourceSetName, dependencies) ->
dependencies.map { artifactModel ->
PackageSearchKotlinMultiplatformDeclaredDependency.Maven(
id = artifactModel.packageId,
declaredVersion = artifactModel.version?.let { NormalizedVersion.fromStringOrNull(it) },
remoteInfo = dependencyInfo[artifactModel.packageId] as? ApiMavenPackage,
declarationIndexes = artifactModel.indexes,
groupId = artifactModel.groupId,
artifactId = artifactModel.artifactId,
variantName = sourceSetName,
configuration = artifactModel.configuration,
icon = dependencyInfo[artifactModel.packageId]?.icon
?: Icons.GRADLE
)
}
}
val sourceSetVariants = compilationModel
.mapKeys { it.key }
.map { (sourceSetName, compilationTargets) ->
PackageSearchKotlinMultiplatformVariant.SourceSet(
name = sourceSetName,
declaredDependencies = declaredSourceSetDependencies[sourceSetName] ?: emptyList(),
attributes = compilationTargets.buildAttributes(),
compatiblePackageTypes = buildPackageTypes {
gradlePackages {
kotlinMultiplatform {
compilationTargets.forEach { compilationTarget ->
when {
compilationTarget is Js -> when (compilationTarget.compiler) {
Js.Compiler.IR -> jsIr()
Js.Compiler.LEGACY -> jsLegacy()
}

compilationTarget is Native -> native(compilationTarget.target)
compilationTarget == MppCompilationInfoModel.Wasm -> wasm()
}
}
when {
Android in compilationTargets -> android()
Jvm in compilationTargets -> jvm()
}
}
}
},
compilerTargets = compilationTargets
)
}

sourceSetVariants + dependenciesBlockVariant.await()
}

}

Loading

0 comments on commit acffb8f

Please sign in to comment.