From 91e937d1045666a7b0f51a2ce8e1d75291bfa2fb Mon Sep 17 00:00:00 2001 From: Lamberto Basti Date: Tue, 30 Jan 2024 16:07:54 +0100 Subject: [PATCH] Update package search cache for offline support The PackageSearchApiPackageCache has been updated to support offline functionality. Refactored the cache system to bypass network calls when operating offline, while optimizing package search parameters. --- .../plugin/core/nitrite/NitriteFilters.kt | 3 + .../PackageSearchApplicationCachesService.kt | 15 +---- .../services/PackageSearchProjectService.kt | 3 + .../plugin/ui/model/ToolWindowViewModel.kt | 2 +- .../plugin/ui/model/tree/TreeViewModel.kt | 7 ++- .../panels/tree/PackageSearchModulesTree.kt | 2 +- .../plugin/utils/ApiPackageCacheEntry.kt | 6 +- .../utils/PackageSearchApiPackageCache.kt | 56 +++++++++++++------ .../messages/packageSearchBundle.properties | 2 +- 9 files changed, 60 insertions(+), 36 deletions(-) diff --git a/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt b/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt index a699a9f0..782af85f 100644 --- a/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt +++ b/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt @@ -19,6 +19,9 @@ object NitriteFilters { fun `in`(path: DocumentPathBuilder, value: Collection): ObjectFilter = `in`(path, value.toTypedArray()) + fun `in`(path: KProperty, value: Collection): ObjectFilter = + ObjectFilters.`in`(path.name, *value.toTypedArray()) + fun `in`(path: String, value: Collection): ObjectFilter = ObjectFilters.`in`(path, *value.toTypedArray()) diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt index c211c253..ce30fd41 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt @@ -16,14 +16,12 @@ import com.jetbrains.packagesearch.plugin.core.nitrite.buildDefaultNitrate import com.jetbrains.packagesearch.plugin.core.nitrite.div import com.jetbrains.packagesearch.plugin.core.utils.PKGSInternalAPI import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModelNodeProcessor -import com.jetbrains.packagesearch.plugin.http.SerializableCachedResponseData import com.jetbrains.packagesearch.plugin.utils.ApiPackageCacheEntry import com.jetbrains.packagesearch.plugin.utils.ApiRepositoryCacheEntry import com.jetbrains.packagesearch.plugin.utils.ApiSearchEntry import com.jetbrains.packagesearch.plugin.utils.KtorDebugLogger import com.jetbrains.packagesearch.plugin.utils.PackageSearchApiPackageCache import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService -import io.ktor.client.engine.cio.CIO import io.ktor.client.engine.java.Java import io.ktor.client.plugins.logging.LogLevel import io.ktor.client.plugins.logging.Logging @@ -67,9 +65,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin private inline fun getRepository(key: String) = cache.getRepository(key) - private val sonatypeCacheRepository - get() = getRepository("sonatype-cache") - private val packagesRepository get() = getRepository("packages") @@ -98,6 +93,7 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin PackageSearchApiPackageCache( apiPackageCache = packagesRepository, searchCache = searchesRepository, + repositoryCache = repositoryCache, apiClient = apiClient, isOnline = it ) @@ -111,15 +107,11 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin ) packagesRepository.createIndex( indexOptions = IndexOptions.indexOptions(IndexType.Unique), - path = ApiPackageCacheEntry::data / ApiPackage::id + path = ApiPackageCacheEntry::packageId ) packagesRepository.createIndex( indexOptions = IndexOptions.indexOptions(IndexType.Unique), - path = ApiPackageCacheEntry::data / ApiPackage::idHash - ) - sonatypeCacheRepository.createIndex( - indexOptions = IndexOptions.indexOptions(IndexType.NonUnique), - path = SerializableCachedResponseData::url + path = ApiPackageCacheEntry::packageIdHash ) } @@ -148,7 +140,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin searchesRepository.removeAll() packagesRepository.removeAll() repositoryCache.removeAll() - sonatypeCacheRepository.removeAll() } } diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt index b53cbcf7..7e6d4ad5 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt @@ -20,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.fus.logOnlyStableToggle import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys +import com.jetbrains.packagesearch.plugin.utils.logDebug import com.jetbrains.packagesearch.plugin.utils.logWarn import com.jetbrains.packagesearch.plugin.utils.nativeModulesFlow import com.jetbrains.packagesearch.plugin.utils.startWithNull @@ -137,6 +138,8 @@ class PackageSearchProjectService( attempt < 3 } .onEach { resetCounter() } + .filter { it.isNotEmpty() } + .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "Total modules -> ${it.size}" } } .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList()) val modulesByBuildFile = modulesStateFlow diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt index 88fd84a5..ada79954 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt @@ -54,7 +54,7 @@ class ToolWindowViewModel(project: Project, private val viewModelScope: Coroutin project.PackageSearchProjectService.packagesBeingDownloadedFlow, project.isProjectImportingFlow, project.service() - .tree + .treeStateFlow .map { !it.isEmpty() } .debounce(250.milliseconds), project.smartModeFlow diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt index b398a450..f5f8668f 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt @@ -7,10 +7,12 @@ import com.intellij.openapi.project.Project import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService +import com.jetbrains.packagesearch.plugin.utils.logDebug import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.retry import kotlinx.coroutines.flow.stateIn import org.jetbrains.jewel.foundation.lazy.SelectableLazyListState @@ -24,13 +26,14 @@ internal class TreeViewModel( viewModelScope: CoroutineScope, ) { - val tree: StateFlow> = combine( + val treeStateFlow: StateFlow> = combine( project.PackageSearchProjectService.modulesStateFlow, project.PackageSearchProjectService.stableOnlyStateFlow ) { modules, stableOnly -> modules.asTree(stableOnly) } .retry() + .onEach { logDebug("${this::class.qualifiedName}#treeStateFlow") { "tree roots -> ${it.roots.size}" } } .stateIn(viewModelScope, SharingStarted.Lazily, emptyTree()) internal val lazyListState = LazyListState() @@ -40,7 +43,7 @@ internal class TreeViewModel( get() = IntelliJApplication.PackageSearchApplicationCachesService.isOnlineFlow fun expandAll() { - treeState.openNodes = tree.value.walkBreadthFirst().map { it.id }.toSet() + treeState.openNodes = treeStateFlow.value.walkBreadthFirst().map { it.id }.toSet() } fun collapseAll() { diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt index 15565fde..f8f3d80b 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt @@ -48,7 +48,7 @@ fun PackageSearchModulesTree( val viewModel: TreeViewModel = viewModel() val knownNodes = remember { mutableSetOf() } - val tree by viewModel.tree.collectAsState() + val tree by viewModel.treeStateFlow.collectAsState() val isOnline by viewModel.isOnline.collectAsState() TreeActionToolbar( isOnline = isOnline, diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt index de483540..4229e1cd 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt @@ -10,7 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest @Serializable data class ApiPackageCacheEntry( - val data: ApiPackage, + val data: ApiPackage?, + val packageId: String, + val packageIdHash: String, @SerialName("_id") val id: Long? = null, val lastUpdate: Instant = Clock.System.now(), ) @@ -31,4 +33,4 @@ data class ApiRepositoryCacheEntry( val lastUpdate: Instant = Clock.System.now(), ) -fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this) \ No newline at end of file +fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this, id, idHash) \ No newline at end of file diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt index 1a4dee82..5e79680e 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt @@ -2,15 +2,12 @@ package com.jetbrains.packagesearch.plugin.utils import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository -import com.jetbrains.packagesearch.plugin.core.nitrite.div import com.jetbrains.packagesearch.plugin.core.nitrite.insert import korlibs.crypto.SHA256 import kotlin.random.Random import kotlin.time.Duration import kotlin.time.Duration.Companion.days -import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.singleOrNull import kotlinx.coroutines.flow.toList import kotlinx.coroutines.sync.Mutex @@ -20,12 +17,14 @@ import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import org.dizitart.no2.objects.ObjectFilter import org.jetbrains.packagesearch.api.v3.ApiPackage +import org.jetbrains.packagesearch.api.v3.ApiRepository import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest class PackageSearchApiPackageCache( private val apiPackageCache: CoroutineObjectRepository, private val searchCache: CoroutineObjectRepository, + private val repositoryCache: CoroutineObjectRepository, private val apiClient: PackageSearchApi, private val maxAge: Duration = Random.nextDouble(0.5, 1.0).days, private val isOnline: Boolean, @@ -37,14 +36,16 @@ class PackageSearchApiPackageCache( getPackages( ids = ids, apiCall = { apiClient.getPackageInfoByIds(it) }, - query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::id, it) } + query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) }, + useHashes = false ) override suspend fun getPackageInfoByIdHashes(ids: Set): Map = getPackages( ids = ids, apiCall = { apiClient.getPackageInfoByIdHashes(it) }, - query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::idHash, it) } + query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) }, + useHashes = true ) override suspend fun searchPackages(request: SearchPackagesRequest): List { @@ -63,30 +64,51 @@ class PackageSearchApiPackageCache( .also { searchCache.insert(ApiSearchEntry(it, sha, request)) } } + override suspend fun getKnownRepositories(): List { + val cached = repositoryCache.find().singleOrNull() + if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnline)) { + return cached.data + } + return if (isOnline) apiClient.getKnownRepositories() + .also { + repositoryCache.removeAll() + repositoryCache.insert(ApiRepositoryCacheEntry(it)) + } + else emptyList() + } + private suspend fun getPackages( ids: Set, apiCall: suspend (Set) -> Map, query: (Set) -> ObjectFilter, + useHashes: Boolean, ): Map = cachesMutex.withLock { if (ids.isEmpty()) return emptyMap() val localDatabaseResults = apiPackageCache.find(query(ids)) - .filter { Clock.System.now() < it.lastUpdate + maxAge } - .map { it.data } + .filter { if (isOnline) Clock.System.now() < it.lastUpdate + maxAge else true } .toList() + val missingIds = ids - when { + useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet() + else -> localDatabaseResults.map { it.packageId }.toSet() + } + + val localDatabaseResultsData = localDatabaseResults + .mapNotNull { it.data } .associateBy { it.id } - val missingIds = ids - localDatabaseResults.keys - if (missingIds.isNotEmpty() && isOnline) { - val networkResults = apiCall(missingIds) - // TODO cache also miss in network to avoid pointless empty query - if (networkResults.isNotEmpty()) { + when { + missingIds.isEmpty() || !isOnline -> localDatabaseResultsData + else -> { + val networkResults = apiCall(missingIds) + .takeIf { it.isNotEmpty() } + ?: return emptyMap() val packageEntries = networkResults.values.map { it.asCacheEntry() } apiPackageCache.remove(NitriteFilters.Object.`in`( - path = ApiPackageCacheEntry::data / ApiPackage::id, - value = packageEntries.map { it.data.id } + path = ApiPackageCacheEntry::packageId, + value = packageEntries.map { it.packageId } )) apiPackageCache.insert(packageEntries) - localDatabaseResults + networkResults - } else localDatabaseResults - } else localDatabaseResults + localDatabaseResultsData + networkResults + } + } } } \ No newline at end of file diff --git a/plugin/src/main/resources/messages/packageSearchBundle.properties b/plugin/src/main/resources/messages/packageSearchBundle.properties index 56a0b1c5..5491048d 100644 --- a/plugin/src/main/resources/messages/packageSearchBundle.properties +++ b/plugin/src/main/resources/messages/packageSearchBundle.properties @@ -205,7 +205,7 @@ packagesearch.ui.util.numberWithThousandsSymbol={0}k packagesearch.version.undefined=Plugin version undefined packagesearch.title.tab=Declared dependencies -packagesearch.cache.clean=Clean Package Search caches +packagesearch.cache.clean=Invalidate Package Search caches packagesearch.toolwindow.loading.syncing=Waiting for package import... packagesearch.toolwindow.loading.downloading=Downloading package data...