Serialization of RealmAny with collections

CHANGELOG.md

@@ -1,4 +1,33 @@
-## 1.11.0-SNAPSHOT (YYYY-MM-DD)
+## 1.12.0-SNAPSHOT (YYYY-MM-DD)
+
+### Breaking Changes
+* None.
+
+### Enhancements
+* None.
+
+### Fixed
+* None.
+
+### Compatibility
+* File format: Generates Realms with file format v23.
+* Realm Studio 13.0.0 or above is required to open Realms created by this version.
+* This release is compatible with the following Kotlin releases:
+  * Kotlin 1.8.0 and above. The K2 compiler is not supported yet.
+  * Ktor 2.1.2 and above.
+  * Coroutines 1.7.0 and above.
+  * AtomicFu 0.18.3 and above.
+  * The new memory model only. See https://github.com/realm/realm-kotlin#kotlin-memory-model-and-coroutine-compatibility
+* Minimum Kbson 0.3.0.
+* Minimum Gradle version: 6.8.3.
+* Minimum Android Gradle Plugin version: 4.1.3.
+* Minimum Android SDK: 16.
+
+### Internal
+* None.
+
+
+## 1.11.0 (2023-09-01)
 
 This release will bump the Realm file format from version 23 to 24. Opening a file with an older format will automatically upgrade it. Downgrading to a previous file format is not possible.
 
@@ -31,9 +60,13 @@ if the content is the same. Custom implementations of these methods will be resp
 * Support for automatic resolution of embedded object constraints during migration through `RealmConfiguration.Builder.migration(migration: AutomaticSchemaMigration, resolveEmbeddedObjectConstraints: Boolean)`. (Issue [#1464](https://github.com/realm/realm-kotlin/issues/1464)
 * Support for collections in `RealmAny`. (Issue [#1434](https://github.com/realm/realm-kotlin/issues/1434))
 * [Sync] Add support for customizing authorization headers and adding additional custom headers to all Atlas App service requests with `AppConfiguration.Builder.authorizationHeaderName()` and `AppConfiguration.Builder.addCustomRequestHeader(...)`. (Issue [#1453](https://github.com/realm/realm-kotlin/pull/1453))
+* [Sync] Added support for manually triggering a reconnect attempt for Device Sync. This is done through a new `App.Sync.reconnect()` method. This method is also now called automatically when a mobile device toggles off airplane mode. (Issue [#1479](https://github.com/realm/realm-kotlin/issues/1479))
 
 ### Fixed
-* None.
+* Rare corruption causing 'Invalid streaming format cookie'-exception. Typically following compact, convert or copying to a new file. (Issue [#1440](https://github.com/realm/realm-kotlin/issues/1440))
+* Compiler error when using Kotlin 1.9.0 and backlinks. (Issue [#1469](https://github.com/realm/realm-kotlin/issues/1469))
+* Leaking `JVMScheduler` instances. In certain circumstances, it could lead to a JNI crash. (Issue [#1463](https://github.com/realm/realm-kotlin/pull/1463))
+* [Sync] Changing a subscriptions query type or query itself will now trigger the `WaitForSync.FIRST_TIME` behaviour, rather than only checking changes to the name. (Issues [#1466](https://github.com/realm/realm-kotlin/issues/1466))
 
 ### Compatibility
 * File format: Generates Realms with file format v24.
@@ -50,7 +83,7 @@ if the content is the same. Custom implementations of these methods will be resp
 * Minimum Android SDK: 16.
 
 ### Internal
-* Updated to Realm Core 13.19.0, commit ea7c5d5e2900b8411a295aea3d1aa56aa55fff1d.
+* Updated to Realm Core 13.20.0, commit c258e2681bca5fb33bbd23c112493817b43bfa86.
 
 
 ## 1.10.2 (2023-07-21)

buildSrc/src/main/kotlin/Config.kt

@@ -62,7 +62,7 @@ val HOST_OS: OperatingSystem = findHostOs()
 
 object Realm {
     val ciBuild = (System.getenv("JENKINS_HOME") != null)
-    const val version = "1.11.0-SNAPSHOT"
+    const val version = "1.12.0-SNAPSHOT"
     const val group = "io.realm.kotlin"
     const val projectUrl = "https://realm.io"
     const val pluginPortalId = "io.realm.kotlin"
@@ -127,7 +127,7 @@ object Versions {
     const val jvmTarget = "1.8"
     // When updating the Kotlin version, also remember to update /examples/min-android-sample/build.gradle.kts
     const val kotlin = "1.8.21" // https://github.com/JetBrains/kotlin and https://kotlinlang.org/docs/releases.html#release-details
-    const val latestKotlin = "1.9.0-Beta" // https://kotlinlang.org/docs/eap.html#build-details
+    const val latestKotlin = "1.9.0" // https://kotlinlang.org/docs/eap.html#build-details
     const val kotlinCompileTesting = "1.5.0" // https://github.com/tschuchortdev/kotlin-compile-testing
     const val ktlint = "0.45.2" // https://github.com/pinterest/ktlint
     const val ktor = "2.1.2" // https://github.com/ktorio/ktor

examples/kmm-sample/gradle.properties

@@ -24,3 +24,4 @@ kotlin.mpp.enableGranularSourceSetsMetadata=true
 kotlin.native.enableDependencyPropagation=false
 
 kotlin.mpp.stability.nowarn=true
+kotlin.mpp.androidSourceSetLayoutVersion1.nowarn=true

examples/kmm-sample/shared/src/commonMain/kotlin/io/realm/example/kmmsample/AllTypes.kt

@@ -17,6 +17,7 @@
 
 package io.realm.example.kmmsample
 
+import io.realm.kotlin.ext.backlinks
 import io.realm.kotlin.ext.realmListOf
 import io.realm.kotlin.types.ObjectId
 import io.realm.kotlin.types.RealmInstant
@@ -83,4 +84,7 @@ class AllTypes : RealmObject {
     var objectIdRealmList: RealmList<ObjectId> = realmListOf(ObjectId.create())
     var objectIdRealmListNullable: RealmList<ObjectId?> = realmListOf(null)
     var objectRealmList: RealmList<AllTypes> = realmListOf()
+
+    // Special types
+    val parent by backlinks(AllTypes::objectField)
 }

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -57,6 +57,7 @@ expect enum class ErrorCode : CodeDescription {
     RLM_ERR_SYNC_CLIENT_RESET_REQUIRED,
     RLM_ERR_SYNC_COMPENSATING_WRITE,
     RLM_ERR_SYNC_CONNECT_FAILED,
+    RLM_ERR_SYNC_CONNECT_TIMEOUT,
     RLM_ERR_SYNC_INVALID_SCHEMA_CHANGE,
     RLM_ERR_SYNC_PERMISSION_DENIED,
     RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED,

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/NativePointer.kt

@@ -43,3 +43,14 @@ public interface NativePointer<T : CapiT> {
      */
     public fun isReleased(): Boolean
 }
+
+/**
+ * Deletes the underlying pointer after executing the lambda block.
+ */
+fun <T : CapiT, R> NativePointer<T>.use(
+    block: (NativePointer<T>) -> R,
+): R = try {
+    block(this)
+} finally {
+    release()
+}

packages/cinterop/src/commonMain/kotlin/io/realm/kotlin/internal/interop/RealmInterop.kt

@@ -68,6 +68,7 @@ interface RealmQueryT : CapiT
 interface RealmCallbackTokenT : CapiT
 interface RealmNotificationTokenT : CapiT
 interface RealmChangesT : CapiT
+interface RealmSchedulerT : CapiT
 
 // Public type aliases binding to internal verbose type safe type definitions. This should allow us
 // to easily change implementation details later on.
@@ -86,6 +87,7 @@ typealias RealmQueryPointer = NativePointer<RealmQueryT>
 typealias RealmCallbackTokenPointer = NativePointer<RealmCallbackTokenT>
 typealias RealmNotificationTokenPointer = NativePointer<RealmNotificationTokenT>
 typealias RealmChangesPointer = NativePointer<RealmChangesT>
+typealias RealmSchedulerPointer = NativePointer<RealmSchedulerT>
 
 // Sync types
 // Pure marker interfaces corresponding to the C-API realm_x_t struct types
@@ -183,6 +185,8 @@ expect object RealmInterop {
     fun realm_config_set_in_memory(config: RealmConfigurationPointer, inMemory: Boolean)
     fun realm_schema_validate(schema: RealmSchemaPointer, mode: SchemaValidationMode): Boolean
 
+    fun realm_create_scheduler(): RealmSchedulerPointer
+    fun realm_create_scheduler(dispatcher: CoroutineDispatcher): RealmSchedulerPointer
     /**
      * Open a realm on the current thread.
      *
@@ -204,7 +208,7 @@ expect object RealmInterop {
      */
     // The dispatcher argument is only used on Native to build a core scheduler dispatching to the
     // dispatcher. The realm itself must also be opened on the same thread
-    fun realm_open(config: RealmConfigurationPointer, dispatcher: CoroutineDispatcher? = null): Pair<LiveRealmPointer, Boolean>
+    fun realm_open(config: RealmConfigurationPointer, scheduler: RealmSchedulerPointer): Pair<LiveRealmPointer, Boolean>
 
     // Opening a Realm asynchronously. Only supported for synchronized realms.
     fun realm_open_synchronized(config: RealmConfigurationPointer): RealmAsyncOpenTaskPointer
@@ -735,6 +739,11 @@ expect object RealmInterop {
         callback: AppCallback<String>
     )
 
+    // Sync Client
+    fun realm_app_sync_client_reconnect(app: RealmAppPointer)
+    fun realm_app_sync_client_has_sessions(app: RealmAppPointer): Boolean
+    fun realm_app_sync_client_wait_for_sessions_to_terminate(app: RealmAppPointer)
+
     // Sync config
     fun realm_config_set_sync_config(
         realmConfiguration: RealmConfigurationPointer,

packages/cinterop/src/jvm/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -54,6 +54,7 @@ actual enum class ErrorCode(override val description: String, override val nativ
     RLM_ERR_SYNC_CLIENT_RESET_REQUIRED("SyncClientResetRequired", realm_errno_e.RLM_ERR_SYNC_CLIENT_RESET_REQUIRED),
     RLM_ERR_SYNC_COMPENSATING_WRITE("CompensatingWrite", realm_errno_e.RLM_ERR_SYNC_COMPENSATING_WRITE),
     RLM_ERR_SYNC_CONNECT_FAILED("SyncConnectFailed", realm_errno_e.RLM_ERR_SYNC_CONNECT_FAILED),
+    RLM_ERR_SYNC_CONNECT_TIMEOUT("SyncConnectTimeout", realm_errno_e.RLM_ERR_SYNC_CONNECT_TIMEOUT),
     RLM_ERR_SYNC_INVALID_SCHEMA_CHANGE("SyncInvalidSchemaChange", realm_errno_e.RLM_ERR_SYNC_INVALID_SCHEMA_CHANGE),
     RLM_ERR_SYNC_PERMISSION_DENIED("SyncPermissionDenied", realm_errno_e.RLM_ERR_SYNC_PERMISSION_DENIED),
     RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED("SyncProtocolInvariantFailed", realm_errno_e.RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED),

packages/cinterop/src/jvm/kotlin/io/realm/kotlin/internal/interop/RealmInterop.kt

@@ -32,7 +32,6 @@ import io.realm.kotlin.internal.interop.sync.SyncSessionResyncMode
 import io.realm.kotlin.internal.interop.sync.SyncUserIdentity
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.CoroutineStart
 import kotlinx.coroutines.launch
 import org.mongodb.kbson.ObjectId
 
@@ -183,22 +182,26 @@ actual object RealmInterop {
         realmc.realm_config_set_in_memory(config.cptr(), inMemory)
     }
 
-    actual fun realm_open(config: RealmConfigurationPointer, dispatcher: CoroutineDispatcher?): Pair<LiveRealmPointer, Boolean> {
+    actual fun realm_create_scheduler(): RealmSchedulerPointer =
+        LongPointerWrapper(realmc.realm_scheduler_make_default())
+
+    actual fun realm_create_scheduler(dispatcher: CoroutineDispatcher): RealmSchedulerPointer =
+        LongPointerWrapper(realmc.realm_create_scheduler(JVMScheduler(dispatcher)))
+
+    actual fun realm_open(
+        config: RealmConfigurationPointer,
+        scheduler: RealmSchedulerPointer,
+    ): Pair<LiveRealmPointer, Boolean> {
         // Configure callback to track if the file was created as part of opening
         var fileCreated = false
         val callback = DataInitializationCallback {
             fileCreated = true
         }
         realm_config_set_data_initialization_function(config, callback)
 
-        // create a custom Scheduler for JVM if a Coroutine Dispatcher is provided other wise
-        // pass null to use the generic one
-        val realmPtr = LongPointerWrapper<LiveRealmT>(
-            realmc.open_realm_with_scheduler(
-                (config as LongPointerWrapper).ptr,
-                if (dispatcher != null) JVMScheduler(dispatcher) else null
-            )
-        )
+        realmc.realm_config_set_scheduler(config.cptr(), scheduler.cptr())
+        val realmPtr = LongPointerWrapper<LiveRealmT>(realmc.realm_open(config.cptr()))
+
         // Ensure that we can read version information, etc.
         realm_begin_read(realmPtr)
         return Pair(realmPtr, fileCreated)
@@ -952,6 +955,9 @@ actual object RealmInterop {
         val deletionCount = LongArray(1)
         val modificationCount = LongArray(1)
         val movesCount = LongArray(1)
+        // Not exposed in SDK yet, but could be used to provide optimized notifications when
+        // collections are cleared.
+        //  https://github.com/realm/realm-kotlin/issues/1498
         val collectionWasCleared = BooleanArray(1)
         val collectionWasDeleted = BooleanArray(1)
 
@@ -1058,8 +1064,10 @@ actual object RealmInterop {
         val deletionStructs = realmc.new_valueArray(deletions[0].toInt())
         val insertionStructs = realmc.new_valueArray(insertions[0].toInt())
         val modificationStructs = realmc.new_valueArray(modifications[0].toInt())
-        // FIXME New in core ... what does it do?
-        val collection_was_cleared = booleanArrayOf(false)
+        // Not exposed in SDK yet, but could be used to provide optimized notifications when
+        // collections are cleared.
+        //  https://github.com/realm/realm-kotlin/issues/1498
+        val collectionWasCleared = booleanArrayOf(false)
         realmc.realm_dictionary_get_changed_keys(
             change.cptr(),
             deletionStructs,
@@ -1068,7 +1076,7 @@ actual object RealmInterop {
             insertions,
             modificationStructs,
             modifications,
-            collection_was_cleared
+            collectionWasCleared
         )
 
         // TODO optimize - integrate within mem allocator?
@@ -1634,6 +1642,17 @@ actual object RealmInterop {
         )
     }
 
+    actual fun realm_app_sync_client_reconnect(app: RealmAppPointer) {
+        realmc.realm_app_sync_client_reconnect(app.cptr())
+    }
+    actual fun realm_app_sync_client_has_sessions(app: RealmAppPointer): Boolean {
+        return realmc.realm_app_sync_client_has_sessions(app.cptr())
+    }
+
+    actual fun realm_app_sync_client_wait_for_sessions_to_terminate(app: RealmAppPointer) {
+        realmc.realm_app_sync_client_wait_for_sessions_to_terminate(app.cptr())
+    }
+
     actual fun realm_sync_config_new(user: RealmUserPointer, partition: String): RealmSyncConfigurationPointer {
         return LongPointerWrapper<RealmSyncConfigT>(realmc.realm_sync_config_new(user.cptr(), partition)).also { ptr ->
             // Stop the session immediately when the Realm is closed, so the lifecycle of the
@@ -2163,13 +2182,8 @@ private class JVMScheduler(dispatcher: CoroutineDispatcher) {
     val scope: CoroutineScope = CoroutineScope(dispatcher)
 
     fun notifyCore(schedulerPointer: Long) {
-        val function: suspend CoroutineScope.() -> Unit = {
+        scope.launch {
             realmc.invoke_core_notify_callback(schedulerPointer)
         }
-        scope.launch(
-            scope.coroutineContext,
-            CoroutineStart.DEFAULT,
-            function
-        )
     }
 }

packages/cinterop/src/nativeDarwin/kotlin/io/realm/kotlin/internal/interop/ErrorCode.kt

@@ -58,6 +58,7 @@ actual enum class ErrorCode(
     RLM_ERR_SYNC_CLIENT_RESET_REQUIRED("SyncClientResetRequired", realm_errno.RLM_ERR_SYNC_CLIENT_RESET_REQUIRED),
     RLM_ERR_SYNC_COMPENSATING_WRITE("CompensatingWrite", realm_errno.RLM_ERR_SYNC_COMPENSATING_WRITE),
     RLM_ERR_SYNC_CONNECT_FAILED("SyncConnectFailed", realm_errno.RLM_ERR_SYNC_CONNECT_FAILED),
+    RLM_ERR_SYNC_CONNECT_TIMEOUT("SyncConnectTimeout", realm_errno.RLM_ERR_SYNC_CONNECT_TIMEOUT),
     RLM_ERR_SYNC_INVALID_SCHEMA_CHANGE("SyncInvalidSchemaChange", realm_errno.RLM_ERR_SYNC_INVALID_SCHEMA_CHANGE),
     RLM_ERR_SYNC_PERMISSION_DENIED("SyncPermissionDenied", realm_errno.RLM_ERR_SYNC_PERMISSION_DENIED),
     RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED("SyncProtocolInvariantFailed", realm_errno.RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED),