241 - Fix missing packages, enhance logging, cache unknown packages

buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt

@@ -41,6 +41,9 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor
     @get:Input
     val packageName = objects.property<String>()
 
+    @get:Input
+    val databaseVersion = objects.property<Int>()
+
     @get:Input
     val objectName = objects.property<String>()
         .convention("PackageSearch")
@@ -90,6 +93,15 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor
                             )
                             .build()
                     )
+                    .addProperty(
+                        PropertySpec.builder("databaseVersion", Int::class)
+                            .getter(
+                                FunSpec.getterBuilder()
+                                    .addStatement("return %L", databaseVersion.get())
+                                    .build()
+                            )
+                            .build()
+                    )
                     .build()
             )
             .build()

plugin/core/build.gradle.kts

@@ -52,6 +52,7 @@ tasks {
         pluginId = pkgsPluginId
         outputDir = generatedDir
         packageName = "com.jetbrains.packagesearch.plugin.core"
+        databaseVersion = 1
     }
     sourcesJar {
         dependsOn(generatePluginDataSources)

plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt

@@ -15,7 +15,7 @@ import kotlin.io.path.div
 class PackageSearchProjectCachesService(private val project: Project) : Disposable {
 
     private val cacheFilePath
-        get() = cachesDirectory / "db-${PackageSearch.pluginVersion}.db"
+        get() = cachesDirectory / "db-v${PackageSearch.databaseVersion}.db"
 
     private val cachesDirectory
         get() = project.getProjectDataPath("caches") / "packagesearch"

plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt

@@ -99,12 +99,11 @@ suspend fun Module.getDeclaredDependencies(): List<PackageSearchGradleDeclaredPa
     val remoteInfo = getPackageInfoByIdHashes(distinctIds.map { ApiPackage.hashPackageId(it) }.toSet())
 
     return declaredDependencies
-        .mapNotNull { declaredDependency ->
+        .map { declaredDependency ->
             PackageSearchGradleDeclaredPackage(
                 id = declaredDependency.packageId,
                 declaredVersion = declaredDependency.version?.let { NormalizedVersion.fromStringOrNull(it) },
-                remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage
-                    ?: return@mapNotNull null,
+                remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage,
                 icon = remoteInfo[declaredDependency.packageId]?.icon
                     ?: IconProvider.Icons.MAVEN,
                 module = declaredDependency.groupId,

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt

@@ -46,7 +46,7 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
 
     companion object {
         private val cacheFilePath
-            get() = appSystemDir / "caches" / "packagesearch" / "db-${PackageSearch.pluginVersion}.db"
+            get() = appSystemDir / "caches" / "packagesearch" / "db-v${PackageSearch.databaseVersion}.db"
     }
 
     @PKGSInternalAPI

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt

@@ -131,7 +131,6 @@ class PackageSearchProjectService(
         .flatMapLatest { moduleProvidersList }
         .retry(5)
         .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } }
-        .modifiedBy(restartFlow) { _, _ -> emptyList() }
         .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
 
     val modulesByBuildFile = modulesStateFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt

@@ -10,9 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 @Serializable
 data class ApiPackageCacheEntry(
-    val data: ApiPackage?,
-    val packageId: String,
-    val packageIdHash: String,
+    val data: ApiPackage? = null,
+    val packageId: String? = null,
+    val packageIdHash: String? = null,
     @SerialName("_id") val id: Long? = null,
     val lastUpdate: Instant = Clock.System.now(),
 )

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt

@@ -3,8 +3,8 @@ package com.jetbrains.packagesearch.plugin.utils
 import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
 import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository
 import com.jetbrains.packagesearch.plugin.core.nitrite.insert
+import com.jetbrains.packagesearch.plugin.core.utils.suspendSafe
 import korlibs.crypto.SHA256
-import kotlin.coroutines.cancellation.CancellationException
 import kotlin.random.Random
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
@@ -16,7 +16,6 @@ import kotlinx.coroutines.sync.withLock
 import kotlinx.datetime.Clock
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import org.dizitart.no2.objects.ObjectFilter
 import org.jetbrains.packagesearch.api.v3.ApiPackage
 import org.jetbrains.packagesearch.api.v3.ApiRepository
 import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi
@@ -36,36 +35,33 @@ class PackageSearchApiPackageCache(
     override suspend fun getPackageInfoByIds(ids: Set<String>) =
         getPackages(
             ids = ids,
-            apiCall = { apiClient.getPackageInfoByIds(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) },
             useHashes = false
         )
 
     override suspend fun getPackageInfoByIdHashes(ids: Set<String>): Map<String, ApiPackage> =
         getPackages(
             ids = ids,
-            apiCall = { apiClient.getPackageInfoByIdHashes(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) },
             useHashes = true
         )
 
     override suspend fun searchPackages(request: SearchPackagesRequest): List<ApiPackage> {
         val sha = SHA256.digest(Json.encodeToString(request).toByteArray()).base64
-        logDebug("${this::class}#searchPackages") { "Searching for packages | searchSha = $sha" }
+        val contextName = "${Random.nextInt()} | ${this::class}#searchPackages"
+        logDebug(contextName) { "Searching for packages | searchSha = $sha" }
         val cachedEntry = searchCache.find(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha))
             .singleOrNull()
         if (cachedEntry != null) {
             val isOffline = !isOnline()
             val isCacheValid = cachedEntry.lastUpdate + maxAge > Clock.System.now()
             if (isOffline || isCacheValid) {
-                logDebug("${this::class}#searchPackages") {
+                logDebug(contextName) {
                     "Using cached search results because `isOffline = $isOffline || isCacheValid = $isCacheValid` | searchSha = $sha"
                 }
                 return cachedEntry.packages
             }
             searchCache.remove(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha))
         }
-        logDebug("${this::class}#searchPackages") { "Fetching search results from the server | searchSha = $sha" }
+        logDebug(contextName) { "Fetching search results from the server | searchSha = $sha" }
         return apiClient.searchPackages(request)
             .also { searchCache.insert(ApiSearchEntry(it, sha, request)) }
     }
@@ -85,41 +81,93 @@ class PackageSearchApiPackageCache(
 
     private suspend fun getPackages(
         ids: Set<String>,
-        apiCall: suspend (Set<String>) -> Map<String, ApiPackage>,
-        query: (Set<String>) -> ObjectFilter,
         useHashes: Boolean,
     ): Map<String, ApiPackage> = cachesMutex.withLock {
         if (ids.isEmpty()) return emptyMap()
-        val localDatabaseResults = apiPackageCache.find(query(ids))
-            .filter { if (isOnline()) Clock.System.now() < it.lastUpdate + maxAge else true }
+        val contextName = "${Random.nextInt()} | ${this::class.qualifiedName}#getPackages"
+        logDebug(contextName) { "Downloading packages | ids.size = ${ids.size}" }
+        val isOnlineStatus = isOnline()
+        val packageIdSelector = when {
+            useHashes -> ApiPackageCacheEntry::packageIdHash
+            else -> ApiPackageCacheEntry::packageId
+        }
+        val apiCall = when {
+            useHashes -> apiClient::getPackageInfoByIdHashes
+            else -> apiClient::getPackageInfoByIds
+        }
+
+        // retrieve the packages from the local database
+        val localDatabaseResults = apiPackageCache
+            .find(NitriteFilters.Object.`in`(packageIdSelector, ids))
+            .filter { if (isOnlineStatus) Clock.System.now() < it.lastUpdate + maxAge else true }
             .toList()
+
+        // evaluate packages that are missing from the local database
         val missingIds = ids - when {
-            useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet()
-            else -> localDatabaseResults.map { it.packageId }.toSet()
+            useHashes -> localDatabaseResults.mapNotNull { it.packageIdHash }.toSet()
+            else -> localDatabaseResults.mapNotNull { it.packageId }.toSet()
         }
+        logDebug(contextName) { "Missing packages | missingIds.size = ${missingIds.size}" }
 
+        // filter away packages that are unknown in our backend
         val localDatabaseResultsData = localDatabaseResults
             .mapNotNull { it.data }
             .associateBy { it.id }
         when {
-            missingIds.isEmpty() || !isOnline() -> localDatabaseResultsData
+            missingIds.isEmpty() || !isOnlineStatus -> {
+                logDebug(contextName) {
+                    "Using cached packages only | isOnline = $isOnlineStatus, localDatabaseResultsData.size = ${localDatabaseResultsData.size}"
+                }
+                localDatabaseResultsData
+            }
+
             else -> {
+                // retrieve the packages from the network
                 val networkResults = runCatching { apiCall(missingIds) }
-                    .onFailure { if (it is CancellationException) throw it }
+                    .suspendSafe()
                     .onFailure { logDebug("${this::class.qualifiedName}#getPackages", it) }
-                    .getOrNull()
-                    ?: emptyMap()
-                if (networkResults.isNotEmpty()) {
-                    val packageEntries = networkResults.values.map { it.asCacheEntry() }
-                    apiPackageCache.remove(
-                        filter = NitriteFilters.Object.`in`(
-                            path = ApiPackageCacheEntry::packageId,
-                            value = packageEntries.map { it.packageId }
+                if (networkResults.isSuccess) {
+                    val packageEntries = networkResults.getOrThrow()
+                        .values
+                        .map { it.asCacheEntry() }
+                    if (packageEntries.isNotEmpty()) {
+                        logDebug(contextName) { "No packages found | missingIds.size = ${missingIds.size}" }
+
+                        // remove the old entries
+                        apiPackageCache.remove(
+                            filter = NitriteFilters.Object.`in`(
+                                path = packageIdSelector,
+                                value = packageEntries.mapNotNull { it.packageId }
+                            )
                         )
-                    )
-                    apiPackageCache.insert(packageEntries)
+                        logDebug(contextName) {
+                            "Removing old entries | packageEntries.size = ${packageEntries.size}"
+                        }
+                    }
+                    // evaluate packages that are missing from our backend
+                    val retrievedPackageIds =
+                        packageEntries.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
+                            .toSet()
+                    val unknownPackages = missingIds.minus(retrievedPackageIds)
+                        .map { id ->
+                            when {
+                                useHashes -> ApiPackageCacheEntry(packageIdHash = id)
+                                else -> ApiPackageCacheEntry(packageId = id)
+                            }
+                        }
+                    logDebug(contextName) {
+                        "New unknown packages | unknownPackages.size = ${unknownPackages.size}"
+                    }
+                    // insert the new entries
+                    val toInsert = packageEntries + unknownPackages
+                    if (toInsert.isNotEmpty()) apiPackageCache.insert(toInsert)
+                }
+                val networkResultsData = networkResults.getOrDefault(emptyMap())
+                logDebug(contextName) {
+                    "Using network results and caches | networkResults.size = ${networkResultsData.size}, " +
+                            "localDatabaseResultsData = ${localDatabaseResultsData.size}"
                 }
-                localDatabaseResultsData + networkResults
+                localDatabaseResultsData + networkResultsData
             }
         }
     }