Update package search cache for offline support

The PackageSearchApiPackageCache has been updated to support offline functionality. Refactored the cache system to bypass network calls when operating offline, while optimizing package search parameters.

nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt

@@ -19,6 +19,9 @@ object NitriteFilters {
         fun `in`(path: DocumentPathBuilder, value: Collection<Any>): ObjectFilter =
             `in`(path, value.toTypedArray())
 
+        fun <T> `in`(path: KProperty<T>, value: Collection<Any>): ObjectFilter =
+            ObjectFilters.`in`(path.name, *value.toTypedArray())
+
         fun `in`(path: String, value: Collection<Any>): ObjectFilter =
             ObjectFilters.`in`(path, *value.toTypedArray())
 

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt

@@ -16,14 +16,12 @@ import com.jetbrains.packagesearch.plugin.core.nitrite.buildDefaultNitrate
 import com.jetbrains.packagesearch.plugin.core.nitrite.div
 import com.jetbrains.packagesearch.plugin.core.utils.PKGSInternalAPI
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModelNodeProcessor
-import com.jetbrains.packagesearch.plugin.http.SerializableCachedResponseData
 import com.jetbrains.packagesearch.plugin.utils.ApiPackageCacheEntry
 import com.jetbrains.packagesearch.plugin.utils.ApiRepositoryCacheEntry
 import com.jetbrains.packagesearch.plugin.utils.ApiSearchEntry
 import com.jetbrains.packagesearch.plugin.utils.KtorDebugLogger
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApiPackageCache
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService
-import io.ktor.client.engine.cio.CIO
 import io.ktor.client.engine.java.Java
 import io.ktor.client.plugins.logging.LogLevel
 import io.ktor.client.plugins.logging.Logging
@@ -67,9 +65,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
     private inline fun <reified T : Any> getRepository(key: String) =
         cache.getRepository<T>(key)
 
-    private val sonatypeCacheRepository
-        get() = getRepository<SerializableCachedResponseData>("sonatype-cache")
-
     private val packagesRepository
         get() = getRepository<ApiPackageCacheEntry>("packages")
 
@@ -98,6 +93,7 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
             PackageSearchApiPackageCache(
                 apiPackageCache = packagesRepository,
                 searchCache = searchesRepository,
+                repositoryCache = repositoryCache,
                 apiClient = apiClient,
                 isOnline = it
             )
@@ -111,15 +107,11 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
         )
         packagesRepository.createIndex(
             indexOptions = IndexOptions.indexOptions(IndexType.Unique),
-            path = ApiPackageCacheEntry::data / ApiPackage::id
+            path = ApiPackageCacheEntry::packageId
         )
         packagesRepository.createIndex(
             indexOptions = IndexOptions.indexOptions(IndexType.Unique),
-            path = ApiPackageCacheEntry::data / ApiPackage::idHash
-        )
-        sonatypeCacheRepository.createIndex(
-            indexOptions = IndexOptions.indexOptions(IndexType.NonUnique),
-            path = SerializableCachedResponseData::url
+            path = ApiPackageCacheEntry::packageIdHash
         )
     }
 
@@ -148,7 +140,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
         searchesRepository.removeAll()
         packagesRepository.removeAll()
         repositoryCache.removeAll()
-        sonatypeCacheRepository.removeAll()
     }
 }
 

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt

@@ -20,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.fus.logOnlyStableToggle
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
 import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext
 import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys
+import com.jetbrains.packagesearch.plugin.utils.logDebug
 import com.jetbrains.packagesearch.plugin.utils.logWarn
 import com.jetbrains.packagesearch.plugin.utils.nativeModulesFlow
 import com.jetbrains.packagesearch.plugin.utils.startWithNull
@@ -137,6 +138,8 @@ class PackageSearchProjectService(
             attempt < 3
         }
         .onEach { resetCounter() }
+        .filter { it.isNotEmpty() }
+        .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "Total modules -> ${it.size}" } }
         .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
 
     val modulesByBuildFile = modulesStateFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt

@@ -54,7 +54,7 @@ class ToolWindowViewModel(project: Project, private val viewModelScope: Coroutin
         project.PackageSearchProjectService.packagesBeingDownloadedFlow,
         project.isProjectImportingFlow,
         project.service<TreeViewModel>()
-            .tree
+            .treeStateFlow
             .map { !it.isEmpty() }
             .debounce(250.milliseconds),
         project.smartModeFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt

@@ -7,10 +7,12 @@ import com.intellij.openapi.project.Project
 import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService
+import com.jetbrains.packagesearch.plugin.utils.logDebug
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.combine
+import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.retry
 import kotlinx.coroutines.flow.stateIn
 import org.jetbrains.jewel.foundation.lazy.SelectableLazyListState
@@ -24,13 +26,14 @@ internal class TreeViewModel(
     viewModelScope: CoroutineScope,
 ) {
 
-    val tree: StateFlow<Tree<TreeItemModel>> = combine(
+    val treeStateFlow: StateFlow<Tree<TreeItemModel>> = combine(
         project.PackageSearchProjectService.modulesStateFlow,
         project.PackageSearchProjectService.stableOnlyStateFlow
     ) { modules, stableOnly ->
         modules.asTree(stableOnly)
     }
         .retry()
+        .onEach { logDebug("${this::class.qualifiedName}#treeStateFlow") { "tree roots -> ${it.roots.size}" } }
         .stateIn(viewModelScope, SharingStarted.Lazily, emptyTree())
 
     internal val lazyListState = LazyListState()
@@ -40,7 +43,7 @@ internal class TreeViewModel(
         get() = IntelliJApplication.PackageSearchApplicationCachesService.isOnlineFlow
 
     fun expandAll() {
-        treeState.openNodes = tree.value.walkBreadthFirst().map { it.id }.toSet()
+        treeState.openNodes = treeStateFlow.value.walkBreadthFirst().map { it.id }.toSet()
     }
 
     fun collapseAll() {

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt

@@ -48,7 +48,7 @@ fun PackageSearchModulesTree(
     val viewModel: TreeViewModel = viewModel()
     val knownNodes = remember { mutableSetOf<PackageSearchModule.Identity>() }
 
-    val tree by viewModel.tree.collectAsState()
+    val tree by viewModel.treeStateFlow.collectAsState()
     val isOnline by viewModel.isOnline.collectAsState()
     TreeActionToolbar(
         isOnline = isOnline,

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt

@@ -10,7 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 @Serializable
 data class ApiPackageCacheEntry(
-    val data: ApiPackage,
+    val data: ApiPackage?,
+    val packageId: String,
+    val packageIdHash: String,
     @SerialName("_id") val id: Long? = null,
     val lastUpdate: Instant = Clock.System.now(),
 )
@@ -31,4 +33,4 @@ data class ApiRepositoryCacheEntry(
     val lastUpdate: Instant = Clock.System.now(),
 )
 
-fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this)
+fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this, id, idHash)

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt

@@ -2,15 +2,12 @@ package com.jetbrains.packagesearch.plugin.utils
 
 import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
 import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository
-import com.jetbrains.packagesearch.plugin.core.nitrite.div
 import com.jetbrains.packagesearch.plugin.core.nitrite.insert
 import korlibs.crypto.SHA256
 import kotlin.random.Random
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
-import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.singleOrNull
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.sync.Mutex
@@ -20,12 +17,14 @@ import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.dizitart.no2.objects.ObjectFilter
 import org.jetbrains.packagesearch.api.v3.ApiPackage
+import org.jetbrains.packagesearch.api.v3.ApiRepository
 import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi
 import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 class PackageSearchApiPackageCache(
     private val apiPackageCache: CoroutineObjectRepository<ApiPackageCacheEntry>,
     private val searchCache: CoroutineObjectRepository<ApiSearchEntry>,
+    private val repositoryCache: CoroutineObjectRepository<ApiRepositoryCacheEntry>,
     private val apiClient: PackageSearchApi,
     private val maxAge: Duration = Random.nextDouble(0.5, 1.0).days,
     private val isOnline: Boolean,
@@ -37,14 +36,16 @@ class PackageSearchApiPackageCache(
         getPackages(
             ids = ids,
             apiCall = { apiClient.getPackageInfoByIds(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::id, it) }
+            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) },
+            useHashes = false
         )
 
     override suspend fun getPackageInfoByIdHashes(ids: Set<String>): Map<String, ApiPackage> =
         getPackages(
             ids = ids,
             apiCall = { apiClient.getPackageInfoByIdHashes(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::idHash, it) }
+            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) },
+            useHashes = true
         )
 
     override suspend fun searchPackages(request: SearchPackagesRequest): List<ApiPackage> {
@@ -63,30 +64,51 @@ class PackageSearchApiPackageCache(
             .also { searchCache.insert(ApiSearchEntry(it, sha, request)) }
     }
 
+    override suspend fun getKnownRepositories(): List<ApiRepository> {
+        val cached = repositoryCache.find().singleOrNull()
+        if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnline)) {
+            return cached.data
+        }
+        return if (isOnline) apiClient.getKnownRepositories()
+            .also {
+                repositoryCache.removeAll()
+                repositoryCache.insert(ApiRepositoryCacheEntry(it))
+            }
+        else emptyList()
+    }
+
     private suspend fun getPackages(
         ids: Set<String>,
         apiCall: suspend (Set<String>) -> Map<String, ApiPackage>,
         query: (Set<String>) -> ObjectFilter,
+        useHashes: Boolean,
     ): Map<String, ApiPackage> = cachesMutex.withLock {
         if (ids.isEmpty()) return emptyMap()
         val localDatabaseResults = apiPackageCache.find(query(ids))
-            .filter { Clock.System.now() < it.lastUpdate + maxAge }
-            .map { it.data }
+            .filter { if (isOnline) Clock.System.now() < it.lastUpdate + maxAge else true }
             .toList()
+        val missingIds = ids - when {
+            useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet()
+            else -> localDatabaseResults.map { it.packageId }.toSet()
+        }
+
+        val localDatabaseResultsData = localDatabaseResults
+            .mapNotNull { it.data }
             .associateBy { it.id }
-        val missingIds = ids - localDatabaseResults.keys
-        if (missingIds.isNotEmpty() && isOnline) {
-            val networkResults = apiCall(missingIds)
-            // TODO cache also miss in network to avoid pointless empty query
-            if (networkResults.isNotEmpty()) {
+        when {
+            missingIds.isEmpty() || !isOnline -> localDatabaseResultsData
+            else -> {
+                val networkResults = apiCall(missingIds)
+                    .takeIf { it.isNotEmpty() }
+                    ?: return emptyMap()
                 val packageEntries = networkResults.values.map { it.asCacheEntry() }
                 apiPackageCache.remove(NitriteFilters.Object.`in`(
-                    path = ApiPackageCacheEntry::data / ApiPackage::id,
-                    value = packageEntries.map { it.data.id }
+                    path = ApiPackageCacheEntry::packageId,
+                    value = packageEntries.map { it.packageId }
                 ))
                 apiPackageCache.insert(packageEntries)
-                localDatabaseResults + networkResults
-            } else localDatabaseResults
-        } else localDatabaseResults
+                localDatabaseResultsData + networkResults
+            }
+        }
     }
 }

plugin/src/main/resources/messages/packageSearchBundle.properties

@@ -205,7 +205,7 @@ packagesearch.ui.util.numberWithThousandsSymbol={0}k
 
 packagesearch.version.undefined=Plugin version undefined
 packagesearch.title.tab=Declared dependencies
-packagesearch.cache.clean=Clean Package Search caches
+packagesearch.cache.clean=Invalidate Package Search caches
 packagesearch.toolwindow.loading.syncing=Waiting for package import...
 packagesearch.toolwindow.loading.downloading=Downloading package data...