Fix race condition in ProgressListenerTest

CHANGELOG.md

@@ -27,9 +27,10 @@
 * Minimum R8: 8.0.34.
 
 ### Internal
-* Update to Ktor 2.3.4.
+* Updated to Ktor 2.3.7.
 * Updated to CMake 3.27.7
-* Updated to Realm Core 13.25.0, commit 71f94d75e25bfc8913fcd93ae8de550b57577a4a.
+* Updated BAAS test server to v2024-01-22.
+* Updated to Realm Core 13.26.0, commit 8b1678096117b886b42a0576e40c75866c496b8f.
 
 
 ## 1.13.1-SNAPSHOT (YYYY-MM-DD)

Jenkinsfile

@@ -35,6 +35,8 @@ runTests = true
 isReleaseBranch = releaseBranches.contains(currentBranch)
 // Manually wipe the workspace before checking out the code. This happens automatically on release branches.
 forceWipeWorkspace = false
+// Whether or not to use platform networking for tests
+usePlatformNetworking = false
 
 // References to Docker containers holding the MongoDB Test server and infrastructure for
 // controlling it.
@@ -191,7 +193,7 @@ pipeline {
                                     "integrationtest",
                                     {
                                         forwardAdbPorts()
-                                        testAndCollect("packages", "cleanAllTests -PsyncUsePlatformNetworking=true -PincludeSdkModules=false connectedAndroidTest")
+                                        testAndCollect("packages", "cleanAllTests -PsyncUsePlatformNetworking=${usePlatformNetworking} -PincludeSdkModules=false connectedAndroidTest")
                                     }
                                 )
                             }
@@ -215,7 +217,7 @@ pipeline {
                     steps {
                         testWithServer([
                             {
-                                testAndCollect("packages", 'cleanAllTests jvmTest -PsyncUsePlatformNetworking=true -PincludeSdkModules=false ')
+                                testAndCollect("packages", "cleanAllTests jvmTest -PsyncUsePlatformNetworking=${usePlatformNetworking} -PincludeSdkModules=false")
                             }
                         ])
                     }
@@ -235,7 +237,7 @@ pipeline {
                     steps {
                         testWithServer([
                             {
-                                testAndCollect("packages", 'cleanAllTests :test-sync:connectedAndroidtest -PsyncUsePlatformNetworking=true -PincludeSdkModules=false -PtestBuildType=debugMinified')
+                                testAndCollect("packages", "cleanAllTests :test-sync:connectedAndroidtest -PsyncUsePlatformNetworking=${usePlatformNetworking} -PincludeSdkModules=false -PtestBuildType=debugMinified")
                             }
                         ])
                         sh 'rm mapping.zip || true'

buildSrc/src/main/kotlin/Config.kt

@@ -131,7 +131,7 @@ object Versions {
     const val latestKotlin = "1.9.20" // https://kotlinlang.org/docs/eap.html#build-details
     const val kotlinCompileTesting = "1.5.0" // https://github.com/tschuchortdev/kotlin-compile-testing
     const val ktlint = "0.45.2" // https://github.com/pinterest/ktlint
-    const val ktor = "2.3.4" // https://github.com/ktorio/ktor
+    const val ktor = "2.3.7" // https://github.com/ktorio/ktor
     const val multidex = "2.0.1" // https://developer.android.com/jetpack/androidx/releases/multidex
     const val nexusPublishPlugin = "1.1.0" // https://github.com/gradle-nexus/publish-plugin
     const val okio = "3.2.0" // https://square.github.io/okio/#releases

dependencies.list

@@ -1,11 +1,11 @@
 # Version of MongoDB Realm used by integration tests
 # See https://github.com/realm/ci/packages/147854 for available versions
-MONGODB_REALM_SERVER=2023-12-15
+MONGODB_REALM_SERVER=2024-01-22
 
 # `BAAS` and `BAAS-UI` projects commit hashes matching MONGODB_REALM_SERVER image version
 # note that the MONGODB_REALM_SERVER image is a nightly build, find the matching commits
 # for that date within the following repositories:
 # https://github.com/10gen/baas/
 # https://github.com/10gen/baas-ui/
-REALM_BAAS_GIT_HASH=47d9f6170ab1ac2aa64e7b5046e85247f3ac6d30
-REALM_BAAS_UI_GIT_HASH=49157ef4a6af1c1de4dfbad5d7d02543776b25eb
+REALM_BAAS_GIT_HASH=97b7445b1634c7a93fbabefc50967b41ce3cc330
+REALM_BAAS_UI_GIT_HASH=f5f5d71e634c2a64d08b2f911e2d7bf91d9cceda

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -68,6 +68,7 @@ expect enum class ErrorCode : CodeDescription {
     RLM_ERR_WRONG_SYNC_TYPE,
     RLM_ERR_SYNC_WRITE_NOT_ALLOWED,
     RLM_ERR_SYNC_LOCAL_CLOCK_BEFORE_EPOCH,
+    RLM_ERR_SYNC_SCHEMA_MIGRATION_ERROR,
     RLM_ERR_SYSTEM_ERROR,
     RLM_ERR_LOGIC,
     RLM_ERR_NOT_SUPPORTED,

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/sync/ProtocolErrorCode.kt

@@ -82,7 +82,9 @@ expect enum class SyncSessionErrorCode : CodeDescription {
     RLM_SYNC_ERR_SESSION_COMPENSATING_WRITE,
     RLM_SYNC_ERR_SESSION_MIGRATE_TO_FLX,
     RLM_SYNC_ERR_SESSION_BAD_PROGRESS,
-    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS;
+    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS,
+    RLM_SYNC_ERR_SESSION_BAD_SCHEMA_VERSION,
+    RLM_SYNC_ERR_SESSION_SCHEMA_VERSION_CHANGED;
 
     companion object {
         internal fun of(nativeValue: Int): SyncSessionErrorCode?

packages/cinterop/src/jvm/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -65,6 +65,7 @@ actual enum class ErrorCode(override val description: String, override val nativ
     RLM_ERR_WRONG_SYNC_TYPE("WrongSyncType", realm_errno_e.RLM_ERR_WRONG_SYNC_TYPE),
     RLM_ERR_SYNC_WRITE_NOT_ALLOWED("SyncWriteNotAllowed", realm_errno_e.RLM_ERR_SYNC_WRITE_NOT_ALLOWED),
     RLM_ERR_SYNC_LOCAL_CLOCK_BEFORE_EPOCH("SyncLocalClockBeforeEpoch", realm_errno_e.RLM_ERR_SYNC_LOCAL_CLOCK_BEFORE_EPOCH),
+    RLM_ERR_SYNC_SCHEMA_MIGRATION_ERROR("SyncSchemaMigrationError", realm_errno_e.RLM_ERR_SYNC_SCHEMA_MIGRATION_ERROR),
     RLM_ERR_SYSTEM_ERROR("SystemError", realm_errno_e.RLM_ERR_SYSTEM_ERROR),
     RLM_ERR_LOGIC("Logic", realm_errno_e.RLM_ERR_LOGIC),
     RLM_ERR_NOT_SUPPORTED("NotSupported", realm_errno_e.RLM_ERR_NOT_SUPPORTED),

packages/cinterop/src/jvm/kotlin/io/realm/kotlin/internal/interop/sync/ProtocolErrorCode.kt

@@ -87,7 +87,9 @@ actual enum class SyncSessionErrorCode(
     RLM_SYNC_ERR_SESSION_COMPENSATING_WRITE("CompensatingWrite", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_COMPENSATING_WRITE),
     RLM_SYNC_ERR_SESSION_MIGRATE_TO_FLX("MigrateToFlexibleSync", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_MIGRATE_TO_FLX),
     RLM_SYNC_ERR_SESSION_BAD_PROGRESS("BadProgress", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_BAD_PROGRESS),
-    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS("RevertToPartitionBasedSync", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_REVERT_TO_PBS);
+    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS("RevertToPartitionBasedSync", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_REVERT_TO_PBS),
+    RLM_SYNC_ERR_SESSION_BAD_SCHEMA_VERSION("BadSchemaVersion", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_BAD_SCHEMA_VERSION),
+    RLM_SYNC_ERR_SESSION_SCHEMA_VERSION_CHANGED("SchemaVersionChanged", realm_sync_errno_session_e.RLM_SYNC_ERR_SESSION_SCHEMA_VERSION_CHANGED);
 
     actual companion object {
         internal actual fun of(nativeValue: Int): SyncSessionErrorCode? =

packages/cinterop/src/nativeDarwin/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -69,6 +69,7 @@ actual enum class ErrorCode(
     RLM_ERR_WRONG_SYNC_TYPE("WrongSyncType", realm_errno.RLM_ERR_WRONG_SYNC_TYPE),
     RLM_ERR_SYNC_WRITE_NOT_ALLOWED("SyncWriteNotAllowed", realm_errno.RLM_ERR_SYNC_WRITE_NOT_ALLOWED),
     RLM_ERR_SYNC_LOCAL_CLOCK_BEFORE_EPOCH("SyncLocalClockBeforeEpoch", realm_errno.RLM_ERR_SYNC_LOCAL_CLOCK_BEFORE_EPOCH),
+    RLM_ERR_SYNC_SCHEMA_MIGRATION_ERROR("SyncSchemaMigrationError", realm_errno.RLM_ERR_SYNC_SCHEMA_MIGRATION_ERROR),
     RLM_ERR_SYSTEM_ERROR("SystemError", realm_errno.RLM_ERR_SYSTEM_ERROR),
     RLM_ERR_LOGIC("Logic", realm_errno.RLM_ERR_LOGIC),
     RLM_ERR_NOT_SUPPORTED("NotSupported", realm_errno.RLM_ERR_NOT_SUPPORTED),

packages/cinterop/src/nativeDarwin/kotlin/io/realm/kotlin/internal/interop/sync/ProtocolErrorCode.kt

@@ -88,7 +88,9 @@ actual enum class SyncSessionErrorCode(
     RLM_SYNC_ERR_SESSION_COMPENSATING_WRITE("CompensatingWrite", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_COMPENSATING_WRITE),
     RLM_SYNC_ERR_SESSION_MIGRATE_TO_FLX("MigrateToFlexibleSync", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_MIGRATE_TO_FLX),
     RLM_SYNC_ERR_SESSION_BAD_PROGRESS("BadProgress", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_BAD_PROGRESS),
-    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS("RevertToPartitionBasedSync", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_REVERT_TO_PBS);
+    RLM_SYNC_ERR_SESSION_REVERT_TO_PBS("RevertToPartitionBasedSync", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_REVERT_TO_PBS),
+    RLM_SYNC_ERR_SESSION_BAD_SCHEMA_VERSION("BadSchemaVersion", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_BAD_SCHEMA_VERSION),
+    RLM_SYNC_ERR_SESSION_SCHEMA_VERSION_CHANGED("SchemaVersionChanged", realm_sync_errno_session.RLM_SYNC_ERR_SESSION_SCHEMA_VERSION_CHANGED);
 
     override val nativeValue: Int = errorCode.value.toInt()
 

packages/test-sync/src/commonMain/kotlin/io/realm/kotlin/test/mongodb/TestApp.kt

@@ -19,6 +19,7 @@
 
 package io.realm.kotlin.test.mongodb
 
+import io.realm.kotlin.Realm
 import io.realm.kotlin.annotations.ExperimentalRealmSerializerApi
 import io.realm.kotlin.internal.interop.RealmInterop
 import io.realm.kotlin.internal.interop.SynchronizableObject
@@ -32,6 +33,8 @@ import io.realm.kotlin.mongodb.App
 import io.realm.kotlin.mongodb.AppConfiguration
 import io.realm.kotlin.mongodb.Credentials
 import io.realm.kotlin.mongodb.User
+import io.realm.kotlin.mongodb.sync.SyncConfiguration
+import io.realm.kotlin.test.mongodb.common.FLEXIBLE_SYNC_SCHEMA
 import io.realm.kotlin.test.mongodb.util.AppAdmin
 import io.realm.kotlin.test.mongodb.util.AppAdminImpl
 import io.realm.kotlin.test.mongodb.util.AppServicesClient
@@ -40,6 +43,7 @@ import io.realm.kotlin.test.mongodb.util.Service
 import io.realm.kotlin.test.mongodb.util.TestAppInitializer.initializeDefault
 import io.realm.kotlin.test.platform.PlatformUtils
 import io.realm.kotlin.test.util.TestHelper
+import io.realm.kotlin.test.util.use
 import kotlinx.coroutines.CloseableCoroutineDispatcher
 import kotlinx.coroutines.CoroutineDispatcher
 import org.mongodb.kbson.ExperimentalKBsonSerializerApi
@@ -117,6 +121,28 @@ open class TestApp private constructor(
         )
     )
 
+    init {
+        // For apps with Flexible Sync, we need to bootstrap all the schemas to work around
+        // https://github.com/realm/realm-core/issues/7297.
+        // So we create a dummy Realm, upload all the schemas and close the Realm again.
+        if (app.configuration.appId.startsWith(TEST_APP_FLEX, ignoreCase = false)) {
+            runBlocking {
+                val user = app.login(Credentials.anonymous())
+                val config = SyncConfiguration.create(user, FLEXIBLE_SYNC_SCHEMA)
+                try {
+                    Realm.open(config).use {
+                        // Using syncSession.uploadAllLocalChanges() seems to just hang forever.
+                        // This is tracked by the above Core issue. Instead use the Sync Progress
+                        // endpoint to signal when the schemas are ready.
+                        pairAdminApp.second.waitForSyncBootstrap()
+                    }
+                } finally {
+                    user.delete()
+                }
+            }
+        }
+    }
+
     fun createUserAndLogin(): User = runBlocking {
         val (email, password) = TestHelper.randomEmail() to "password1234"
         emailPasswordAuth.registerUser(email, password).run {

packages/test-sync/src/commonMain/kotlin/io/realm/kotlin/test/mongodb/common/AsymmetricSchemas.kt

@@ -0,0 +1,63 @@
+package io.realm.kotlin.test.mongodb.common
+
+import io.realm.kotlin.ext.realmListOf
+import io.realm.kotlin.types.AsymmetricRealmObject
+import io.realm.kotlin.types.EmbeddedRealmObject
+import io.realm.kotlin.types.RealmList
+import io.realm.kotlin.types.RealmObject
+import io.realm.kotlin.types.annotations.PersistedName
+import io.realm.kotlin.types.annotations.PrimaryKey
+import org.mongodb.kbson.BsonObjectId
+import org.mongodb.kbson.ObjectId
+
+class DeviceParent : RealmObject {
+    @PersistedName("_id")
+    @PrimaryKey
+    var id: ObjectId = BsonObjectId()
+    var device: Device? = null
+}
+
+class Measurement : AsymmetricRealmObject {
+    @PersistedName("_id")
+    @PrimaryKey
+    var id: ObjectId = BsonObjectId()
+    var type: String = "temperature"
+    var value: Float = 0.0f
+    var device: Device? = null
+    var backups: RealmList<BackupDevice> = realmListOf()
+}
+
+class BackupDevice() : EmbeddedRealmObject {
+    constructor(name: String, serialNumber: String) : this() {
+        this.name = name
+        this.serialNumber = serialNumber
+    }
+    var name: String = ""
+    var serialNumber: String = ""
+}
+
+class Device() : EmbeddedRealmObject {
+    constructor(name: String, serialNumber: String) : this() {
+        this.name = name
+        this.serialNumber = serialNumber
+    }
+    var name: String = ""
+    var serialNumber: String = ""
+    var backupDevice: BackupDevice? = null
+}
+
+class AsymmetricA : AsymmetricRealmObject {
+    @PrimaryKey
+    var _id: ObjectId = ObjectId()
+    var child: EmbeddedB? = null
+}
+
+class EmbeddedB : EmbeddedRealmObject {
+    var child: StandardC? = null
+}
+
+class StandardC : RealmObject {
+    @PrimaryKey
+    var _id: ObjectId = ObjectId()
+    var name: String = ""
+}

packages/test-sync/src/commonTest/kotlin/io/realm/kotlin/test/mongodb/common/Schema.kt → packages/test-sync/src/commonMain/kotlin/io/realm/kotlin/test/mongodb/common/Schema.kt

@@ -29,9 +29,9 @@ import io.realm.kotlin.entities.sync.flx.FlexEmbeddedObject
 import io.realm.kotlin.entities.sync.flx.FlexParentObject
 
 private val ASYMMETRIC_SCHEMAS = setOf(
-    AsymmetricSyncTests.AsymmetricA::class,
-    AsymmetricSyncTests.EmbeddedB::class,
-    AsymmetricSyncTests.StandardC::class,
+    AsymmetricA::class,
+    EmbeddedB::class,
+    StandardC::class,
     Measurement::class,
 )
 private val DEFAULT_SCHEMAS = setOf(
@@ -52,4 +52,7 @@ private val DEFAULT_SCHEMAS = setOf(
 )
 
 val PARTITION_BASED_SCHEMA = DEFAULT_SCHEMAS
+// Amount of schema classes that should be created on the server. EmbeddedRealmObjects are not
+// included in this count
+val FLEXIBLE_SYNC_SCHEMA_COUNT = 11
 val FLEXIBLE_SYNC_SCHEMA = DEFAULT_SCHEMAS + ASYMMETRIC_SCHEMAS

packages/test-sync/src/commonMain/kotlin/io/realm/kotlin/test/mongodb/util/AppAdmin.kt

@@ -18,7 +18,9 @@ package io.realm.kotlin.test.mongodb.util
 
 import io.realm.kotlin.mongodb.sync.SyncMode
 import io.realm.kotlin.mongodb.sync.SyncSession
+import kotlinx.coroutines.delay
 import kotlinx.serialization.json.JsonObject
+import kotlin.time.Duration.Companion.seconds
 
 /**
  * Wrapper around App Services Server Admin functions needed for tests.
@@ -101,6 +103,11 @@ interface AppAdmin {
      */
     suspend fun deleteDocuments(database: String, clazz: String, query: String): JsonObject?
 
+    /**
+     * Wait for Sync bootstrap to complete for all model classes.
+     */
+    suspend fun waitForSyncBootstrap()
+
     fun closeClient()
 }
 
@@ -201,6 +208,19 @@ class AppAdminImpl(
             app.deleteDocument(database, clazz, query)
         }
 
+    override suspend fun waitForSyncBootstrap() {
+        baasClient.run {
+            var counter = 30
+            while (!app.initialSyncComplete() && counter > 0) {
+                delay(1.seconds)
+                counter--
+            }
+            if (!app.initialSyncComplete()) {
+                throw IllegalStateException("Test server did not finish bootstrapping sync in time.")
+            }
+        }
+    }
+
     override fun closeClient() {
         baasClient.closeClient()
     }

packages/test-sync/src/commonMain/kotlin/io/realm/kotlin/test/mongodb/util/AppServicesClient.kt

@@ -39,6 +39,7 @@ import io.realm.kotlin.internal.platform.runBlocking
 import io.realm.kotlin.mongodb.sync.SyncMode
 import io.realm.kotlin.test.mongodb.SyncServerConfig
 import io.realm.kotlin.test.mongodb.TEST_APP_CLUSTER_NAME
+import io.realm.kotlin.test.mongodb.common.FLEXIBLE_SYNC_SCHEMA_COUNT
 import io.realm.kotlin.test.mongodb.util.TestAppInitializer.initialize
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.withContext
@@ -54,6 +55,7 @@ import kotlinx.serialization.json.JsonNull
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.add
+import kotlinx.serialization.json.boolean
 import kotlinx.serialization.json.buildJsonArray
 import kotlinx.serialization.json.buildJsonObject
 import kotlinx.serialization.json.int
@@ -639,6 +641,42 @@ class AppServicesClient(
             )
         }
 
+    suspend fun BaasApp.initialSyncComplete(): Boolean {
+        return withContext(dispatcher) {
+            try {
+                httpClient.typedRequest<JsonObject>(
+                    Get,
+                    "$url/sync/progress"
+                ).let { obj: JsonObject ->
+                    val statuses: JsonElement = obj["progress"]!!
+                    when (statuses) {
+                        is JsonObject -> {
+                            if (statuses.keys.isEmpty()) {
+                                // It might take a few seconds to register the Schemas, so treat
+                                // "empty" progress as initial sync not being complete (as we always
+                                // have at least one pre-defined schema).
+                                false
+                            }
+                            val bootstrapComplete: List<Boolean> = statuses.keys.map { schemaClass ->
+                                statuses[schemaClass]!!.jsonObject["complete"]?.jsonPrimitive?.boolean == true
+                            }
+                            bootstrapComplete.all { it } && statuses.size == FLEXIBLE_SYNC_SCHEMA_COUNT
+                        }
+                        else -> false
+                    }
+                }
+            } catch (ex: IllegalStateException) {
+                if (ex.message!!.contains("there are no mongodb/atlas services with provided sync state")) {
+                    // If the network returns this error, it means that Sync is not enabled for this app,
+                    // in that case, just report success.
+                    true
+                } else {
+                    throw ex
+                }
+            }
+        }
+    }
+
     private suspend fun BaasApp.getLocalUserPassProviderId(): String =
         withContext(dispatcher) {
             httpClient.typedRequest<JsonArray>(