241 - Update package search cache for offline support

nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/NitriteFilters.kt

@@ -19,6 +19,9 @@ object NitriteFilters {
         fun `in`(path: DocumentPathBuilder, value: Collection<Any>): ObjectFilter =
             `in`(path, value.toTypedArray())
 
+        fun <T> `in`(path: KProperty<T>, value: Collection<Any>): ObjectFilter =
+            ObjectFilters.`in`(path.name, *value.toTypedArray())
+
         fun `in`(path: String, value: Collection<Any>): ObjectFilter =
             ObjectFilters.`in`(path, *value.toTypedArray())
 

plugin/maven/src/main/kotlin/com/jetbrains/packagesearch/plugin/maven/MavenUtils.kt

@@ -40,8 +40,6 @@ import org.jetbrains.idea.maven.project.MavenProjectsManager
 import org.jetbrains.packagesearch.api.v3.ApiPackage
 import org.jetbrains.packagesearch.api.v3.ApiRepository
 import org.jetbrains.packagesearch.api.v3.search.buildPackageTypes
-import org.jetbrains.packagesearch.api.v3.search.javaApi
-import org.jetbrains.packagesearch.api.v3.search.javaRuntime
 import org.jetbrains.packagesearch.api.v3.search.jvmMavenPackages
 import org.jetbrains.packagesearch.maven.POM_XML_NAMESPACE
 import org.jetbrains.packagesearch.maven.ProjectObjectModel
@@ -63,10 +61,16 @@ val commonScopes = listOf("compile", "provided", "runtime", "test", "system", "i
 
 val Project.mavenImportFlow
     get() = messageBus.flow(MavenImportListener.TOPIC) {
-        MavenImportListener { _, _ ->
-            trySend(Unit)
+        object : MavenImportListener {
+            override fun importFinished(
+                importedProjects: MutableCollection<MavenProject>,
+                newModules: MutableList<Module>,
+            ) {
+                trySend(Unit)
+            }
         }
     }
+
 context(ProjectContext)
 fun getModuleChangesFlow(pomPath: Path): Flow<Unit> = merge(
     watchExternalFileChanges(mavenSettingsFilePath),

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt

@@ -16,14 +16,12 @@ import com.jetbrains.packagesearch.plugin.core.nitrite.buildDefaultNitrate
 import com.jetbrains.packagesearch.plugin.core.nitrite.div
 import com.jetbrains.packagesearch.plugin.core.utils.PKGSInternalAPI
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModelNodeProcessor
-import com.jetbrains.packagesearch.plugin.http.SerializableCachedResponseData
 import com.jetbrains.packagesearch.plugin.utils.ApiPackageCacheEntry
 import com.jetbrains.packagesearch.plugin.utils.ApiRepositoryCacheEntry
 import com.jetbrains.packagesearch.plugin.utils.ApiSearchEntry
 import com.jetbrains.packagesearch.plugin.utils.KtorDebugLogger
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApiPackageCache
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService
-import io.ktor.client.engine.cio.CIO
 import io.ktor.client.engine.java.Java
 import io.ktor.client.plugins.logging.LogLevel
 import io.ktor.client.plugins.logging.Logging
@@ -67,9 +65,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
     private inline fun <reified T : Any> getRepository(key: String) =
         cache.getRepository<T>(key)
 
-    private val sonatypeCacheRepository
-        get() = getRepository<SerializableCachedResponseData>("sonatype-cache")
-
     private val packagesRepository
         get() = getRepository<ApiPackageCacheEntry>("packages")
 
@@ -98,6 +93,7 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
             PackageSearchApiPackageCache(
                 apiPackageCache = packagesRepository,
                 searchCache = searchesRepository,
+                repositoryCache = repositoryCache,
                 apiClient = apiClient,
                 isOnline = it
             )
@@ -111,15 +107,11 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
         )
         packagesRepository.createIndex(
             indexOptions = IndexOptions.indexOptions(IndexType.Unique),
-            path = ApiPackageCacheEntry::data / ApiPackage::id
+            path = ApiPackageCacheEntry::packageId
         )
         packagesRepository.createIndex(
             indexOptions = IndexOptions.indexOptions(IndexType.Unique),
-            path = ApiPackageCacheEntry::data / ApiPackage::idHash
-        )
-        sonatypeCacheRepository.createIndex(
-            indexOptions = IndexOptions.indexOptions(IndexType.NonUnique),
-            path = SerializableCachedResponseData::url
+            path = ApiPackageCacheEntry::packageIdHash
         )
     }
 
@@ -148,7 +140,6 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
         searchesRepository.removeAll()
         packagesRepository.removeAll()
         repositoryCache.removeAll()
-        sonatypeCacheRepository.removeAll()
     }
 }
 

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt

@@ -20,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.fus.logOnlyStableToggle
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
 import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext
 import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys
+import com.jetbrains.packagesearch.plugin.utils.logDebug
 import com.jetbrains.packagesearch.plugin.utils.logWarn
 import com.jetbrains.packagesearch.plugin.utils.nativeModulesFlow
 import com.jetbrains.packagesearch.plugin.utils.startWithNull
@@ -137,6 +138,8 @@ class PackageSearchProjectService(
             attempt < 3
         }
         .onEach { resetCounter() }
+        .filter { it.isNotEmpty() }
+        .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "Total modules -> ${it.size}" } }
         .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
 
     val modulesByBuildFile = modulesStateFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/ToolWindowViewModel.kt

@@ -54,7 +54,7 @@ class ToolWindowViewModel(project: Project, private val viewModelScope: Coroutin
         project.PackageSearchProjectService.packagesBeingDownloadedFlow,
         project.isProjectImportingFlow,
         project.service<TreeViewModel>()
-            .tree
+            .treeStateFlow
             .map { !it.isEmpty() }
             .debounce(250.milliseconds),
         project.smartModeFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/model/tree/TreeViewModel.kt

@@ -7,10 +7,12 @@ import com.intellij.openapi.project.Project
 import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchProjectService
+import com.jetbrains.packagesearch.plugin.utils.logDebug
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.combine
+import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.retry
 import kotlinx.coroutines.flow.stateIn
 import org.jetbrains.jewel.foundation.lazy.SelectableLazyListState
@@ -24,13 +26,14 @@ internal class TreeViewModel(
     viewModelScope: CoroutineScope,
 ) {
 
-    val tree: StateFlow<Tree<TreeItemModel>> = combine(
+    val treeStateFlow: StateFlow<Tree<TreeItemModel>> = combine(
         project.PackageSearchProjectService.modulesStateFlow,
         project.PackageSearchProjectService.stableOnlyStateFlow
     ) { modules, stableOnly ->
         modules.asTree(stableOnly)
     }
         .retry()
+        .onEach { logDebug("${this::class.qualifiedName}#treeStateFlow") { "tree roots -> ${it.roots.size}" } }
         .stateIn(viewModelScope, SharingStarted.Lazily, emptyTree())
 
     internal val lazyListState = LazyListState()
@@ -40,7 +43,7 @@ internal class TreeViewModel(
         get() = IntelliJApplication.PackageSearchApplicationCachesService.isOnlineFlow
 
     fun expandAll() {
-        treeState.openNodes = tree.value.walkBreadthFirst().map { it.id }.toSet()
+        treeState.openNodes = treeStateFlow.value.walkBreadthFirst().map { it.id }.toSet()
     }
 
     fun collapseAll() {

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/ui/panels/tree/PackageSearchModulesTree.kt

@@ -48,7 +48,7 @@ fun PackageSearchModulesTree(
     val viewModel: TreeViewModel = viewModel()
     val knownNodes = remember { mutableSetOf<PackageSearchModule.Identity>() }
 
-    val tree by viewModel.tree.collectAsState()
+    val tree by viewModel.treeStateFlow.collectAsState()
     val isOnline by viewModel.isOnline.collectAsState()
     TreeActionToolbar(
         isOnline = isOnline,

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt

@@ -10,7 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 @Serializable
 data class ApiPackageCacheEntry(
-    val data: ApiPackage,
+    val data: ApiPackage?,
+    val packageId: String,
+    val packageIdHash: String,
     @SerialName("_id") val id: Long? = null,
     val lastUpdate: Instant = Clock.System.now(),
 )
@@ -31,4 +33,4 @@ data class ApiRepositoryCacheEntry(
     val lastUpdate: Instant = Clock.System.now(),
 )
 
-fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this)
+fun ApiPackage.asCacheEntry() = ApiPackageCacheEntry(this, id, idHash)

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt

@@ -2,15 +2,12 @@ package com.jetbrains.packagesearch.plugin.utils
 
 import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
 import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository
-import com.jetbrains.packagesearch.plugin.core.nitrite.div
 import com.jetbrains.packagesearch.plugin.core.nitrite.insert
 import korlibs.crypto.SHA256
 import kotlin.random.Random
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
-import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.singleOrNull
 import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.sync.Mutex
@@ -20,12 +17,14 @@ import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.dizitart.no2.objects.ObjectFilter
 import org.jetbrains.packagesearch.api.v3.ApiPackage
+import org.jetbrains.packagesearch.api.v3.ApiRepository
 import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi
 import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 class PackageSearchApiPackageCache(
     private val apiPackageCache: CoroutineObjectRepository<ApiPackageCacheEntry>,
     private val searchCache: CoroutineObjectRepository<ApiSearchEntry>,
+    private val repositoryCache: CoroutineObjectRepository<ApiRepositoryCacheEntry>,
     private val apiClient: PackageSearchApi,
     private val maxAge: Duration = Random.nextDouble(0.5, 1.0).days,
     private val isOnline: Boolean,
@@ -37,14 +36,16 @@ class PackageSearchApiPackageCache(
         getPackages(
             ids = ids,
             apiCall = { apiClient.getPackageInfoByIds(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::id, it) }
+            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) },
+            useHashes = false
         )
 
     override suspend fun getPackageInfoByIdHashes(ids: Set<String>): Map<String, ApiPackage> =
         getPackages(
             ids = ids,
             apiCall = { apiClient.getPackageInfoByIdHashes(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::data / ApiPackage::idHash, it) }
+            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) },
+            useHashes = true
         )
 
     override suspend fun searchPackages(request: SearchPackagesRequest): List<ApiPackage> {
@@ -63,30 +64,51 @@ class PackageSearchApiPackageCache(
             .also { searchCache.insert(ApiSearchEntry(it, sha, request)) }
     }
 
+    override suspend fun getKnownRepositories(): List<ApiRepository> {
+        val cached = repositoryCache.find().singleOrNull()
+        if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnline)) {
+            return cached.data
+        }
+        return if (isOnline) apiClient.getKnownRepositories()
+            .also {
+                repositoryCache.removeAll()
+                repositoryCache.insert(ApiRepositoryCacheEntry(it))
+            }
+        else emptyList()
+    }
+
     private suspend fun getPackages(
         ids: Set<String>,
         apiCall: suspend (Set<String>) -> Map<String, ApiPackage>,
         query: (Set<String>) -> ObjectFilter,
+        useHashes: Boolean,
     ): Map<String, ApiPackage> = cachesMutex.withLock {
         if (ids.isEmpty()) return emptyMap()
         val localDatabaseResults = apiPackageCache.find(query(ids))
-            .filter { Clock.System.now() < it.lastUpdate + maxAge }
-            .map { it.data }
+            .filter { if (isOnline) Clock.System.now() < it.lastUpdate + maxAge else true }
             .toList()
+        val missingIds = ids - when {
+            useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet()
+            else -> localDatabaseResults.map { it.packageId }.toSet()
+        }
+
+        val localDatabaseResultsData = localDatabaseResults
+            .mapNotNull { it.data }
             .associateBy { it.id }
-        val missingIds = ids - localDatabaseResults.keys
-        if (missingIds.isNotEmpty() && isOnline) {
-            val networkResults = apiCall(missingIds)
-            // TODO cache also miss in network to avoid pointless empty query
-            if (networkResults.isNotEmpty()) {
+        when {
+            missingIds.isEmpty() || !isOnline -> localDatabaseResultsData
+            else -> {
+                val networkResults = apiCall(missingIds)
+                    .takeIf { it.isNotEmpty() }
+                    ?: return emptyMap()
                 val packageEntries = networkResults.values.map { it.asCacheEntry() }
                 apiPackageCache.remove(NitriteFilters.Object.`in`(
-                    path = ApiPackageCacheEntry::data / ApiPackage::id,
-                    value = packageEntries.map { it.data.id }
+                    path = ApiPackageCacheEntry::packageId,
+                    value = packageEntries.map { it.packageId }
                 ))
                 apiPackageCache.insert(packageEntries)
-                localDatabaseResults + networkResults
-            } else localDatabaseResults
-        } else localDatabaseResults
+                localDatabaseResultsData + networkResults
+            }
+        }
     }
 }

plugin/src/main/resources/messages/packageSearchBundle.properties

@@ -205,7 +205,7 @@ packagesearch.ui.util.numberWithThousandsSymbol={0}k
 
 packagesearch.version.undefined=Plugin version undefined
 packagesearch.title.tab=Declared dependencies
-packagesearch.cache.clean=Clean Package Search caches
+packagesearch.cache.clean=Invalidate Package Search caches
 packagesearch.toolwindow.loading.syncing=Waiting for package import...
 packagesearch.toolwindow.loading.downloading=Downloading package data...