Skip to content

Commit

Permalink
232 - Improved services lazyness and reduced HTTP client retry attemp…
Browse files Browse the repository at this point in the history 
…ts. (#75)

* Improved services lazyness and reduced HTTP client retry attempts.

* Remove unused gradleIdentityPathOrNull utility

This commit removes the unused 'gradleIdentityPathOrNull' utility from the Utils.kt file. This method was previously used to retrieve Gradle module identity path, but is no longer needed. The appearance and readability of the code have significantly improved as a result.

* Refactor PackageSearchApiPackageCache handling

The code for handling the network results in the PackageSearchApiPackageCache class has been refactored. Instead of performing a removal of old entries followed by an insertion of new entries, the code now performs an update operation for each new entry. This makes the handling of network results more efficient and concise.
  • Loading branch information
@lamba92
lamba92 authored Feb 19, 2024
1 parent ea9af91 commit 377f26a
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 81 deletions.
2 changes: 1 addition & 1 deletion package-search-api-models
Submodule package-search-api-models updated 3 files
+19 −0 http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/PackageSearchApi.kt
+36 −40 http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/PackageSearchApiClient.kt
+26 −1 http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/Utils.kt
108 changes: 76 additions & 32 deletions plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/Utils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import com.intellij.openapi.vfs.VirtualFileEvent
import com.intellij.openapi.vfs.VirtualFileListener
import com.intellij.openapi.vfs.VirtualFileManager
import com.intellij.openapi.vfs.newvfs.events.VFileEvent
import com.intellij.openapi.wm.ToolWindowManager
import com.intellij.openapi.wm.ex.ToolWindowManagerListener
import com.intellij.util.application
import com.intellij.util.messages.MessageBus
import com.intellij.util.messages.Topic
Expand All @@ -44,7 +46,6 @@ import kotlinx.coroutines.flow.FlowCollector
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.callbackFlow
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.merge
Expand All @@ -68,6 +69,40 @@ fun <T : Any, R> MessageBus.flow(
awaitClose { connection.disconnect() }
}

sealed interface FlowEvent<T> {

@JvmInline
value class Added<T>(val item: T) : FlowEvent<T>

@JvmInline
value class Removed<T>(val item: T) : FlowEvent<T>

@JvmInline
value class Initial<T>(val items: List<T>) : FlowEvent<T>
}

fun <T : Any, R> MessageBus.bufferFlow(
topic: Topic<T>,
initialValue: (() -> List<R>)? = null,
listener: ProducerScope<FlowEvent<R>>.() -> T,
) = channelFlow {
val buffer = mutableSetOf<R>()
flow(topic, listener).onEach { event ->
when (event) {
is FlowEvent.Added -> buffer.add(event.item)
is FlowEvent.Removed -> buffer.remove(event.item)
is FlowEvent.Initial -> {
buffer.clear()
buffer.addAll(event.items)
}
}
send(buffer.toList())
}
.launchIn(this)
initialValue?.invoke()?.let { send(it) }
awaitClose()
}

val filesChangedEventFlow: Flow<List<VFileEvent>>
get() = callbackFlow {
val disposable = Disposer.newDisposable()
Expand Down Expand Up @@ -173,43 +208,52 @@ fun <T> Flow<T>.replayOn(vararg replayFlows: Flow<*>) = channelFlow {
merge(*replayFlows).collect { mutex.withLock { last?.let { send(it) } } }
}

val Project.fileOpenedFlow: Flow<List<VirtualFile>>
get() {
val flow = flow {
val buffer: MutableList<VirtualFile> = FileEditorManager.getInstance(this@fileOpenedFlow).openFiles
.toMutableList()
emit(buffer.toList())
messageBus.flow(FileEditorManagerListener.FILE_EDITOR_MANAGER) {
object : FileEditorManagerListener {
override fun fileOpened(source: FileEditorManager, file: VirtualFile) {
trySend(FileEditorEvent.FileOpened(file))
}

override fun fileClosed(source: FileEditorManager, file: VirtualFile) {
trySend(FileEditorEvent.FileClosed(file))
}
}
}.collect {
when (it) {
is FileEditorEvent.FileClosed -> buffer.remove(it.file)
is FileEditorEvent.FileOpened -> buffer.add(it.file)
}
emit(buffer.toList())
}
fun Project.toolWindowOpenedFlow(toolWindowId: String): Flow<Boolean> = callbackFlow {
val manager = ToolWindowManager.getInstance(this@toolWindowOpenedFlow)
val toolWindow = manager.getToolWindow(toolWindowId)

// Initial state
trySend(toolWindow?.isVisible ?: false)

val listener = object : ToolWindowManagerListener {
override fun stateChanged(toolWindowManager: ToolWindowManager) {
trySend(manager.getToolWindow(toolWindowId)?.isVisible ?: false)
}
return flow.withInitialValue(FileEditorManager.getInstance(this@fileOpenedFlow).openFiles.toList())
}

internal sealed interface FileEditorEvent {
// Register the listener
val connection = messageBus.connect()
connection.subscribe(ToolWindowManagerListener.TOPIC, listener)

val file: VirtualFile
// Cleanup on close
awaitClose { connection.disconnect() }
}

@JvmInline
value class FileOpened(override val file: VirtualFile) : FileEditorEvent
// Usage:
// val toolWindowFlow = project.toolWindowOpenedFlow("YourToolWindowId")
// toolWindowFlow.collect { isOpen ->
// println("Tool window is open: $isOpen")
// }


val Project.fileOpenedFlow
get() = messageBus.bufferFlow(
topic = FileEditorManagerListener.FILE_EDITOR_MANAGER,
initialValue = { FileEditorManager.getInstance(this).openFiles.toList() }
) {
object : FileEditorManagerListener {
override fun fileOpened(source: FileEditorManager, file: VirtualFile) {
trySend(FlowEvent.Added(file))
}

@JvmInline
value class FileClosed(override val file: VirtualFile) : FileEditorEvent
}
override fun fileClosed(source: FileEditorManager, file: VirtualFile) {
trySend(FlowEvent.Removed(file))
}
}
}

val Project.project
get() = this

val <T : Any> ExtensionPointName<T>.availableExtensionsFlow: FlowWithInitialValue<List<T>>
get() {
Expand Down
3 changes: 2 additions & 1 deletion ...kotlin/com/jetbrains/packagesearch/plugin/gradle/PackageSearchProjectResolverExtension.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import com.intellij.openapi.externalSystem.model.project.ModuleData
import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel.Configuration
import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel.Dependency
import com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleJavaModel
import com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleModelBuilder
import java.nio.file.Paths
import org.gradle.tooling.model.idea.IdeaModule
import org.jetbrains.plugins.gradle.service.project.AbstractProjectResolverExtension
Expand All @@ -15,7 +16,7 @@ class PackageSearchProjectResolverExtension : AbstractProjectResolverExtension()
setOf(PackageSearchGradleJavaModel::class.java)

override fun getToolingExtensionsClasses() =
setOf(com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleModelBuilder::class.java)
setOf(PackageSearchGradleModelBuilder::class.java)

private inline fun <reified T> IdeaModule.getExtraProject(): T? =
resolverCtx.getExtraProject(this@getExtraProject, T::class.java)
Expand Down
6 changes: 0 additions & 6 deletions plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/Utils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,6 @@ val Module.isGradleSourceSet: Boolean
return ExternalSystemApiUtil.getExternalModuleType(this) == GradleConstants.GRADLE_SOURCE_SET_MODULE_TYPE_KEY
}

val Module.gradleIdentityPathOrNull: String?
get() = CachedModuleDataFinder.getInstance(project)
.findMainModuleData(this)
?.data
?.gradleIdentityPathOrNull

suspend fun Project.awaitExternalSystemInitialization() = suspendCoroutine {
ExternalProjectsManager.getInstance(this@awaitExternalSystemInitialization)
.runWhenInitialized { it.resume(Unit) }
Expand Down
43 changes: 29 additions & 14 deletions ...rc/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication
import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService
import com.jetbrains.packagesearch.plugin.core.utils.fileOpenedFlow
import com.jetbrains.packagesearch.plugin.core.utils.replayOn
import com.jetbrains.packagesearch.plugin.core.utils.toolWindowOpenedFlow
import com.jetbrains.packagesearch.plugin.fus.logOnlyStableToggle
import com.jetbrains.packagesearch.plugin.ui.model.packageslist.modifiedBy
import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext
import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys
Expand All @@ -40,6 +40,7 @@ import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.consumeAsFlow
import kotlinx.coroutines.flow.debounce
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.flatMapLatest
import kotlinx.coroutines.flow.flatMapMerge
Expand Down Expand Up @@ -71,7 +72,7 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch
val isProjectExecutingSyncStateFlow = PackageSearchModuleBaseTransformerUtils.extensionsFlow
.map { it.map { it.getSyncStateFlow(project) } }
.flatMapLatest { combine(it) { it.all { it } } }
.stateIn(coroutineScope, SharingStarted.Eagerly, false)
.stateIn(coroutineScope, SharingStarted.Lazily, false)

private val knownRepositoriesStateFlow = timer(12.hours) {
IntelliJApplication.PackageSearchApplicationCachesService
Expand Down Expand Up @@ -100,7 +101,7 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch
val packagesBeingDownloadedFlow = context.getLoadingFLow()
.distinctUntilChanged()
.onEach { logDebug("${this::class.qualifiedName}#packagesBeingDownloadedFlow") { "$it" } }
.stateIn(coroutineScope, SharingStarted.Eagerly, false)
.stateIn(coroutineScope, SharingStarted.Lazily, false)

private val moduleProvidersList
get() = combine(
Expand Down Expand Up @@ -128,15 +129,23 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch
.flatMapLatest { moduleProvidersList }
.retry(5)
.onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } }
.stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
.stateIn(coroutineScope, SharingStarted.Lazily, emptyList())

val modulesByBuildFile = modulesStateFlow
.map { it.associateBy { it.buildFilePath }.filterNotNullKeys() }
.stateIn(coroutineScope, SharingStarted.Eagerly, emptyMap())
.stateIn(coroutineScope, SharingStarted.Lazily, emptyMap())

val modulesByIdentity = modulesStateFlow
.map { it.associateBy { it.identity } }
.stateIn(coroutineScope, SharingStarted.Eagerly, emptyMap())
.stateIn(coroutineScope, SharingStarted.Lazily, emptyMap())

private val openedBuildFiles = combine(
project.fileOpenedFlow,
modulesByBuildFile.map { it.keys }
) { openedFiles, buildFiles ->
openedFiles.filter { it.toNioPathOrNull()?.let { it in buildFiles } ?: false }
}
.shareIn(coroutineScope, SharingStarted.Lazily, 0)

init {

Expand All @@ -148,8 +157,18 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch
}
.launchIn(coroutineScope)

IntelliJApplication.PackageSearchApplicationCachesService
.isOnlineFlow
combine(
openedBuildFiles.map { it.isEmpty() },
project.toolWindowOpenedFlow("Package Search")
) { noOpenedFiles, toolWindowOpened -> noOpenedFiles || !toolWindowOpened }
.flatMapLatest {
// if the tool window is not opened and there are no opened build files,
// we don't need to do anything, and we turn off the isOnlineFlow
when {
it -> IntelliJApplication.PackageSearchApplicationCachesService.isOnlineFlow
else -> emptyFlow()
}
}
.filter { it }
.onEach { restart() }
.retry {
Expand All @@ -158,12 +177,8 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch
}
.launchIn(coroutineScope)

combine(
project.fileOpenedFlow,
modulesByBuildFile.map { it.keys }
) { openedFiles, buildFiles ->
openedFiles.filter { it.toNioPathOrNull()?.let { it in buildFiles } ?: false }
}

openedBuildFiles
.filter { it.isNotEmpty() }
.replayOn(stableOnlyStateFlow)
.flatMapMerge { it.asFlow() }
Expand Down
4 changes: 2 additions & 2 deletions ...n/kotlin/com/jetbrains/packagesearch/plugin/ui/model/packageslist/PackageListViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class PackageListViewModel(private val project: Project) : Disposable {
combine(listOf(selectedModuleIdsSharedFlow.map { it.size == 1 }, isOnline)) {
it.all { it }
}
.stateIn(viewModelScope, SharingStarted.Eagerly, true)
.stateIn(viewModelScope, SharingStarted.WhileSubscribed(), true)

private val selectedModulesFlow = combine(
selectedModuleIdsSharedFlow,
Expand Down Expand Up @@ -216,7 +216,7 @@ class PackageListViewModel(private val project: Project) : Disposable {
}
}
.retry()
.stateIn(viewModelScope, SharingStarted.Eagerly, emptyList())
.stateIn(viewModelScope, SharingStarted.WhileSubscribed(), emptyList())

private suspend fun PackageSearchModule.Base.getSearchQuery(
searchQuery: String,
Expand Down
56 changes: 31 additions & 25 deletions .../src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,21 @@ class PackageSearchApiPackageCache(

override suspend fun getKnownRepositories(): List<ApiRepository> {
val cached = repositoryCache.find().singleOrNull()
if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnline())) {
val isOnlineStatus = isOnline()
if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnlineStatus)) {
return cached.data
}
return if (isOnline()) apiClient.getKnownRepositories()
.also {
repositoryCache.removeAll()
repositoryCache.insert(ApiRepositoryCacheEntry(it))
}
else emptyList()
return when {
isOnlineStatus -> runCatching { apiClient.getKnownRepositories() }
.suspendSafe()
.onSuccess {
repositoryCache.removeAll()
repositoryCache.insert(ApiRepositoryCacheEntry(it))
}
.getOrDefault(cached?.data ?: emptyList())

else -> emptyList()
}
}

private suspend fun getPackages(
Expand Down Expand Up @@ -127,26 +133,12 @@ class PackageSearchApiPackageCache(
.suspendSafe()
.onFailure { logDebug("${this::class.qualifiedName}#getPackages", it) }
if (networkResults.isSuccess) {
val packageEntries = networkResults.getOrThrow()
val cacheEntriesFromNetwork = networkResults.getOrThrow()
.values
.map { it.asCacheEntry() }
if (packageEntries.isNotEmpty()) {
logDebug(contextName) { "No packages found | missingIds.size = ${missingIds.size}" }

// remove the old entries
apiPackageCache.remove(
filter = NitriteFilters.Object.`in`(
path = packageIdSelector,
value = packageEntries.mapNotNull { it.packageId }
)
)
logDebug(contextName) {
"Removing old entries | packageEntries.size = ${packageEntries.size}"
}
}
// evaluate packages that are missing from our backend
val retrievedPackageIds =
packageEntries.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
cacheEntriesFromNetwork.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
.toSet()
val unknownPackages = missingIds.minus(retrievedPackageIds)
.map { id ->
Expand All @@ -159,8 +151,22 @@ class PackageSearchApiPackageCache(
"New unknown packages | unknownPackages.size = ${unknownPackages.size}"
}
// insert the new entries
val toInsert = packageEntries + unknownPackages
if (toInsert.isNotEmpty()) apiPackageCache.insert(toInsert)
val toInsert = cacheEntriesFromNetwork + unknownPackages
if (toInsert.isNotEmpty()) {
toInsert.forEach { insert ->
apiPackageCache.update(
filter = NitriteFilters.Object.eq(
path = packageIdSelector,
value = when {
useHashes -> insert.packageIdHash
else -> insert.packageId
}
),
update = insert,
upsert = true
)
}
}
}
val networkResultsData = networkResults.getOrDefault(emptyMap())
logDebug(contextName) {
Expand Down

0 comments on commit 377f26a

Please sign in to comment.