Fix missing packages, enhance logging, cache unknown packages (#69)

Updated GradleUtils to enhance processing of remote package info. Enhanced logging for better debuggability. Modified handling of packages unknown in backend ensuring that they are properly tracked. Introduced database version control. Changes result in improved reliability and easier issue tracking.

buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt

@@ -41,6 +41,9 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor
     @get:Input
     val packageName = objects.property<String>()
 
+    @get:Input
+    val databaseVersion = objects.property<Int>()
+
     @get:Input
     val objectName = objects.property<String>()
         .convention("PackageSearch")
@@ -90,6 +93,15 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor
                             )
                             .build()
                     )
+                    .addProperty(
+                        PropertySpec.builder("databaseVersion", Int::class)
+                            .getter(
+                                FunSpec.getterBuilder()
+                                    .addStatement("return %L", databaseVersion.get())
+                                    .build()
+                            )
+                            .build()
+                    )
                     .build()
             )
             .build()

plugin/core/build.gradle.kts

@@ -52,6 +52,7 @@ tasks {
         pluginId = pkgsPluginId
         outputDir = generatedDir
         packageName = "com.jetbrains.packagesearch.plugin.core"
+        databaseVersion = 1
     }
     sourcesJar {
         dependsOn(generatePluginDataSources)

plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt

@@ -15,7 +15,7 @@ import kotlin.io.path.div
 class PackageSearchProjectCachesService(private val project: Project) : Disposable {
 
     private val cacheFilePath
-        get() = cachesDirectory / "db-${PackageSearch.pluginVersion}.db"
+        get() = cachesDirectory / "db-v${PackageSearch.databaseVersion}.db"
 
     private val cachesDirectory
         get() = project.getProjectDataPath("caches") / "packagesearch"

plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt

@@ -99,12 +99,11 @@ suspend fun Module.getDeclaredDependencies(): List<PackageSearchGradleDeclaredPa
     val remoteInfo = getPackageInfoByIdHashes(distinctIds.map { ApiPackage.hashPackageId(it) }.toSet())
 
     return declaredDependencies
-        .mapNotNull { declaredDependency ->
+        .map { declaredDependency ->
             PackageSearchGradleDeclaredPackage(
                 id = declaredDependency.packageId,
                 declaredVersion = declaredDependency.version?.let { NormalizedVersion.fromStringOrNull(it) },
-                remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage
-                    ?: return@mapNotNull null,
+                remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage,
                 icon = remoteInfo[declaredDependency.packageId]?.icon
                     ?: IconProvider.Icons.MAVEN,
                 module = declaredDependency.groupId,

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt

@@ -46,7 +46,7 @@ class PackageSearchApplicationCachesService(private val coroutineScope: Coroutin
 
     companion object {
         private val cacheFilePath
-            get() = appSystemDir / "caches" / "packagesearch" / "db-${PackageSearch.pluginVersion}.db"
+            get() = appSystemDir / "caches" / "packagesearch" / "db-v${PackageSearch.databaseVersion}.db"
     }
 
     @PKGSInternalAPI

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt

@@ -131,7 +131,6 @@ class PackageSearchProjectService(
         .flatMapLatest { moduleProvidersList }
         .retry(5)
         .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } }
-        .modifiedBy(restartFlow) { _, _ -> emptyList() }
         .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
 
     val modulesByBuildFile = modulesStateFlow

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt

@@ -10,9 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest
 
 @Serializable
 data class ApiPackageCacheEntry(
-    val data: ApiPackage?,
-    val packageId: String,
-    val packageIdHash: String,
+    val data: ApiPackage? = null,
+    val packageId: String? = null,
+    val packageIdHash: String? = null,
     @SerialName("_id") val id: Long? = null,
     val lastUpdate: Instant = Clock.System.now(),
 )

plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt

@@ -3,8 +3,8 @@ package com.jetbrains.packagesearch.plugin.utils
 import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
 import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository
 import com.jetbrains.packagesearch.plugin.core.nitrite.insert
+import com.jetbrains.packagesearch.plugin.core.utils.suspendSafe
 import korlibs.crypto.SHA256
-import kotlin.coroutines.cancellation.CancellationException
 import kotlin.random.Random
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
@@ -16,7 +16,6 @@ import kotlinx.coroutines.sync.withLock
 import kotlinx.datetime.Clock
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import org.dizitart.no2.objects.ObjectFilter
 import org.jetbrains.packagesearch.api.v3.ApiPackage
 import org.jetbrains.packagesearch.api.v3.ApiRepository
 import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi
@@ -36,36 +35,33 @@ class PackageSearchApiPackageCache(
     override suspend fun getPackageInfoByIds(ids: Set<String>) =
         getPackages(
             ids = ids,
-            apiCall = { apiClient.getPackageInfoByIds(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) },
             useHashes = false
         )
 
     override suspend fun getPackageInfoByIdHashes(ids: Set<String>): Map<String, ApiPackage> =
         getPackages(
             ids = ids,
-            apiCall = { apiClient.getPackageInfoByIdHashes(it) },
-            query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) },
             useHashes = true
         )
 
     override suspend fun searchPackages(request: SearchPackagesRequest): List<ApiPackage> {
         val sha = SHA256.digest(Json.encodeToString(request).toByteArray()).base64
-        logDebug("${this::class}#searchPackages") { "Searching for packages | searchSha = $sha" }
+        val contextName = "${Random.nextInt()} | ${this::class}#searchPackages"
+        logDebug(contextName) { "Searching for packages | searchSha = $sha" }
         val cachedEntry = searchCache.find(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha))
             .singleOrNull()
         if (cachedEntry != null) {
             val isOffline = !isOnline()
             val isCacheValid = cachedEntry.lastUpdate + maxAge > Clock.System.now()
             if (isOffline || isCacheValid) {
-                logDebug("${this::class}#searchPackages") {
+                logDebug(contextName) {
                     "Using cached search results because `isOffline = $isOffline || isCacheValid = $isCacheValid` | searchSha = $sha"
                 }
                 return cachedEntry.packages
             }
             searchCache.remove(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha))
         }
-        logDebug("${this::class}#searchPackages") { "Fetching search results from the server | searchSha = $sha" }
+        logDebug(contextName) { "Fetching search results from the server | searchSha = $sha" }
         return apiClient.searchPackages(request)
             .also { searchCache.insert(ApiSearchEntry(it, sha, request)) }
     }
@@ -85,41 +81,93 @@ class PackageSearchApiPackageCache(
 
     private suspend fun getPackages(
         ids: Set<String>,
-        apiCall: suspend (Set<String>) -> Map<String, ApiPackage>,
-        query: (Set<String>) -> ObjectFilter,
         useHashes: Boolean,
     ): Map<String, ApiPackage> = cachesMutex.withLock {
         if (ids.isEmpty()) return emptyMap()
-        val localDatabaseResults = apiPackageCache.find(query(ids))
-            .filter { if (isOnline()) Clock.System.now() < it.lastUpdate + maxAge else true }
+        val contextName = "${Random.nextInt()} | ${this::class.qualifiedName}#getPackages"
+        logDebug(contextName) { "Downloading packages | ids.size = ${ids.size}" }
+        val isOnlineStatus = isOnline()
+        val packageIdSelector = when {
+            useHashes -> ApiPackageCacheEntry::packageIdHash
+            else -> ApiPackageCacheEntry::packageId
+        }
+        val apiCall = when {
+            useHashes -> apiClient::getPackageInfoByIdHashes
+            else -> apiClient::getPackageInfoByIds
+        }
+
+        // retrieve the packages from the local database
+        val localDatabaseResults = apiPackageCache
+            .find(NitriteFilters.Object.`in`(packageIdSelector, ids))
+            .filter { if (isOnlineStatus) Clock.System.now() < it.lastUpdate + maxAge else true }
             .toList()
+
+        // evaluate packages that are missing from the local database
         val missingIds = ids - when {
-            useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet()
-            else -> localDatabaseResults.map { it.packageId }.toSet()
+            useHashes -> localDatabaseResults.mapNotNull { it.packageIdHash }.toSet()
+            else -> localDatabaseResults.mapNotNull { it.packageId }.toSet()
         }
+        logDebug(contextName) { "Missing packages | missingIds.size = ${missingIds.size}" }
 
+        // filter away packages that are unknown in our backend
         val localDatabaseResultsData = localDatabaseResults
             .mapNotNull { it.data }
             .associateBy { it.id }
         when {
-            missingIds.isEmpty() || !isOnline() -> localDatabaseResultsData
+            missingIds.isEmpty() || !isOnlineStatus -> {
+                logDebug(contextName) {
+                    "Using cached packages only | isOnline = $isOnlineStatus, localDatabaseResultsData.size = ${localDatabaseResultsData.size}"
+                }
+                localDatabaseResultsData
+            }
+
             else -> {
+                // retrieve the packages from the network
                 val networkResults = runCatching { apiCall(missingIds) }
-                    .onFailure { if (it is CancellationException) throw it }
+                    .suspendSafe()
                     .onFailure { logDebug("${this::class.qualifiedName}#getPackages", it) }
-                    .getOrNull()
-                    ?: emptyMap()
-                if (networkResults.isNotEmpty()) {
-                    val packageEntries = networkResults.values.map { it.asCacheEntry() }
-                    apiPackageCache.remove(
-                        filter = NitriteFilters.Object.`in`(
-                            path = ApiPackageCacheEntry::packageId,
-                            value = packageEntries.map { it.packageId }
+                if (networkResults.isSuccess) {
+                    val packageEntries = networkResults.getOrThrow()
+                        .values
+                        .map { it.asCacheEntry() }
+                    if (packageEntries.isNotEmpty()) {
+                        logDebug(contextName) { "No packages found | missingIds.size = ${missingIds.size}" }
+
+                        // remove the old entries
+                        apiPackageCache.remove(
+                            filter = NitriteFilters.Object.`in`(
+                                path = packageIdSelector,
+                                value = packageEntries.mapNotNull { it.packageId }
+                            )
                         )
-                    )
-                    apiPackageCache.insert(packageEntries)
+                        logDebug(contextName) {
+                            "Removing old entries | packageEntries.size = ${packageEntries.size}"
+                        }
+                    }
+                    // evaluate packages that are missing from our backend
+                    val retrievedPackageIds =
+                        packageEntries.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
+                            .toSet()
+                    val unknownPackages = missingIds.minus(retrievedPackageIds)
+                        .map { id ->
+                            when {
+                                useHashes -> ApiPackageCacheEntry(packageIdHash = id)
+                                else -> ApiPackageCacheEntry(packageId = id)
+                            }
+                        }
+                    logDebug(contextName) {
+                        "New unknown packages | unknownPackages.size = ${unknownPackages.size}"
+                    }
+                    // insert the new entries
+                    val toInsert = packageEntries + unknownPackages
+                    if (toInsert.isNotEmpty()) apiPackageCache.insert(toInsert)
+                }
+                val networkResultsData = networkResults.getOrDefault(emptyMap())
+                logDebug(contextName) {
+                    "Using network results and caches | networkResults.size = ${networkResultsData.size}, " +
+                            "localDatabaseResultsData = ${localDatabaseResultsData.size}"
                 }
-                localDatabaseResultsData + networkResults
+                localDatabaseResultsData + networkResultsData
             }
         }
     }