Bump core 14.10.4

.github/workflows/include-static-analysis.yml

@@ -35,6 +35,7 @@ jobs:
         run: ./gradlew ktlintCheck
 
       - name: Stash Ktlint results
+        if: always()
         run: |
           rm -rf /tmp/ktlint
           rm -rf /tmp/detekt
@@ -50,6 +51,7 @@ jobs:
 
       - name: Publish Ktlint results
         uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: Ktlint Analyzer report
           path: /tmp/ktlint/*
@@ -85,7 +87,8 @@ jobs:
       - name: Run Detekt
         run: ./gradlew detekt
 
-      - name: Stash Detekt results    
+      - name: Stash Detekt results   
+        if: always() 
         run: |
           rm -rf /tmp/detekt
           mkdir /tmp/detekt
@@ -99,6 +102,7 @@ jobs:
 
       - name: Publish Detekt results    
         uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: Detekt Analyzer report
           path: /tmp/detekt/*

.github/workflows/pr.yml

@@ -122,7 +122,7 @@ jobs:
           key: jni-linux-lib-${{ needs.check-cache.outputs.packages-sha }}
 
       - name: Setup Java 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: ${{ vars.VERSION_JAVA_DISTRIBUTION }}
           java-version: ${{ vars.VERSION_JAVA }}
@@ -473,7 +473,7 @@ jobs:
           echo '#!/bin/bash\nccache clang++ "$@"%"' > /usr/local/bin/ccache-clang++          
 
       - name: Setup Android SDK
-        uses: android-actions/setup-android@v2
+        uses: android-actions/setup-android@v3
 
       - name: Install NDK
         run: sdkmanager --install "ndk;${{ env.NDK_VERSION }}"
@@ -547,7 +547,9 @@ jobs:
         uses: actions/setup-java@v4
         with:
           distribution: ${{ vars.VERSION_JAVA_DISTRIBUTION }}
-          java-version: ${{ vars.VERSION_JAVA }}
+          java-version: |
+            17
+            ${{ vars.VERSION_JAVA }}
 
       - name: Setup Gradle and task/dependency caching
         uses: gradle/actions/setup-gradle@v3
@@ -602,9 +604,13 @@ jobs:
           echo '#!/bin/bash\nccache clang++ "$@"%"' > /usr/local/bin/ccache-clang++          
 
       - name: Setup Android SDK
-        uses: android-actions/setup-android@v2
+        env: 
+          JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}
+        uses: android-actions/setup-android@v3
 
       - name: Install NDK
+        env: 
+          JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}
         run: sdkmanager --install "ndk;${{ env.NDK_VERSION }}"
 
       - name: Build Android Base Test Apk

CHANGELOG.md

@@ -4,10 +4,28 @@
 * None.
 
 ### Enhancements
-* None.
+* Reduce the size of the local transaction log produced by creating objects, improving the performance of insertion-heavy transactions (Core issue [realm/realm-core#7734](https://github.com/realm/realm-core/pull/7734)).
+* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in MQL or the Query::between() method when you build the query. (Core issue [realm/realm-core#7785](https://github.com/realm/realm-core/pull/7785))
+* [Sync] Report the originating error that caused a client reset to occur. (Core issue [realm/realm-core#6154](https://github.com/realm/realm-core/issues/6154)).
+* [Sync] It is no longer an error to set a base url for an App with a trailing slash - for example, `https://services.cloud.mongodb.com/` instead of `https://services.cloud.mongodb.com` - before this change that would result in a 404 error from the server (Core issue [realm/realm-core#7791](https://github.com/realm/realm-core/pull/7791)).
+* [Sync] On Windows devices Device Sync will additionally look up SSL certificates in the Windows Trusted Root Certification Authorities certificate store when establishing a connection. (Core issue [realm/realm-core#7882](https://github.com/realm/realm-core/pull/7882))
 
 ### Fixed
-* None.
+* Comparing a numeric property with an argument list containing a string would throw. (Core issue [realm/realm-core#7714](https://github.com/realm/realm-core/issues/7714), since v2.0.0).
+* After compacting, a file upgrade would be triggered. This could cause loss of data if schema mode is SoftResetFile (Core issue [realm/realm-core#7747](https://github.com/realm/realm-core/issues/7747), since v1.15.0).
+* Encrypted files on Windows had a maximum size of 2GB even on x64 due to internal usage of `off_t`, which is a 32-bit type on 64-bit Windows (Core issue [realm/realm-core#7698](https://github.com/realm/realm-core/pull/7698)).
+* The encryption code no longer behaves differently depending on the system page size, which should entirely eliminate a recurring source of bugs related to copying encrypted Realm files between platforms with different page sizes. One known outstanding bug was ([RNET-1141](https://github.com/realm/realm-dotnet/issues/3592)), where opening files on a system with a larger page size than the writing system would attempt to read sections of the file which had never been written to (Core issue [realm/realm-core#7698](https://github.com/realm/realm-core/pull/7698)).
+* There were several complicated scenarios which could result in stale reads from encrypted files in multiprocess scenarios. These were very difficult to hit and would typically lead to a crash, either due to an assertion failure or DecryptionFailure being thrown (Core issue [realm/realm-core#7698](https://github.com/realm/realm-core/pull/7698), since v1.8.0).
+* Encrypted files have some benign data races where we can memcpy a block of memory while another thread is writing to a limited range of it. It is logically impossible to ever read from that range when this happens, but Thread Sanitizer quite reasonably complains about this. We now perform a slower operations when running with TSan which avoids this benign race (Core issue [realm/realm-core#7698](https://github.com/realm/realm-core/pull/7698)).
+* Tokenizing strings for full-text search could pass values outside the range [-1, 255] to `isspace()`, which is undefined behavior (Core issue [realm/realm-core#7698](https://github.com/realm/realm-core/pull/7698), since the introduction of FTS).
+* Clearing a List of RealmAnys in an upgraded file would lead to an assertion failing (Core issue [realm/realm-core#7771](https://github.com/realm/realm-core/issues/7771), since v1.15.0)
+* You could get unexpected merge results when assigning to a nested collection (Core issue [realm/realm-core#7809](https://github.com/realm/realm-core/issues/7809), since v1.15.0)
+* Fixed removing backlinks from the wrong objects if the link came from a nested list, nested dictionary, top-level dictionary, or list of mixed, and the source table had more than 256 objects. This could manifest as `array_backlink.cpp:112: Assertion failed: int64_t(value >> 1) == key.value` when removing an object. (Core issue [realm/realm-core#7594](https://github.com/realm/realm-core/issues/7594), since Core v11 for dictionaries)
+* Fixed the collapse/rejoin of clusters which contained nested collections with links. This could manifest as `array.cpp:319: Array::move() Assertion failed: begin <= end [2, 1]` when removing an object. (Core issue [realm/realm-core#7839](https://github.com/realm/realm-core/issues/7839), since the introduction of nested collections in v1.15.0)
+* [Sync] Fix some client resets (such as migrating to flexible sync) potentially failing with AutoClientResetFailed if a new client reset condition (such as rolling back a flexible sync migration) occurred before the first one completed. (Core issue [realm/realm-core#7542](https://github.com/realm/realm-core/pull/7542), since v1.9.0)
+* [Sync] Fixed a change of mode from Strong to All when removing links from an embedded object that links to a tombstone. This affects sync apps that use embedded objects which have a `Lst<Mixed>` that contains a link to another top level object which has been deleted by another sync client (creating a tombstone locally). In this particular case, the switch would cause any remaining link removals to recursively delete the destination object if there were no other links to it. (Core issue [realm/realm-core#7828](https://github.com/realm/realm-core/issues/7828), since v1.15.0)
+* [Sync] `SyncSession.uploadAllLocalChanges` was inconsistent in how it handled commits which did not produce any changesets to upload. Previously it would sometimes complete immediately if all commits waiting to be uploaded were empty, and at other times it would wait for a server roundtrip. It will now always complete immediately. (Core issue [realm/realm-core#7796](https://github.com/realm/realm-core/pull/7796)).
+* [Sync] Sync client can crash if a session is resumed while the session is being suspended. (Core issue [realm/realm-core#7860](https://github.com/realm/realm-core/issues/7860), since v1.0.0)
 
 ### Compatibility
 * File format: Generates Realms with file format v24 (reads and upgrades file format v10 or later).
@@ -25,7 +43,7 @@
 * Minimum R8: 8.0.34.
 
 ### Internal
-* None.
+* Updated to Realm Core 14.10.4 commit 4f83c590c4340dd7760d5f070e2e81613eb536aa.
 
 
 ## 2.1.0 (2024-07-12)

packages/library-sync/src/commonMain/kotlin/io/realm/kotlin/mongodb/exceptions/SyncExceptions.kt

@@ -68,6 +68,7 @@ public open class UnrecoverableSyncException internal constructor(message: Strin
  * Thrown when the type of sync used by the server does not match the one used by the client, i.e.
  * the server and client disagrees whether to use Partition-based or Flexible Sync.
  */
+@Suppress("DEPRECATION")
 public class WrongSyncTypeException internal constructor(message: String) :
     UnrecoverableSyncException(message)
 

packages/library-sync/src/commonMain/kotlin/io/realm/kotlin/mongodb/internal/RealmSyncUtils.kt

@@ -17,7 +17,6 @@ import io.realm.kotlin.mongodb.exceptions.FunctionExecutionException
 import io.realm.kotlin.mongodb.exceptions.InvalidCredentialsException
 import io.realm.kotlin.mongodb.exceptions.ServiceException
 import io.realm.kotlin.mongodb.exceptions.SyncException
-import io.realm.kotlin.mongodb.exceptions.UnrecoverableSyncException
 import io.realm.kotlin.mongodb.exceptions.UserAlreadyConfirmedException
 import io.realm.kotlin.mongodb.exceptions.UserAlreadyExistsException
 import io.realm.kotlin.mongodb.exceptions.UserNotFoundException
@@ -91,19 +90,26 @@ internal fun convertSyncError(syncError: SyncError): SyncException {
             syncError.compensatingWrites,
             syncError.isFatal
         )
+
         ErrorCode.RLM_ERR_SYNC_PROTOCOL_INVARIANT_FAILED,
         ErrorCode.RLM_ERR_SYNC_PROTOCOL_NEGOTIATION_FAILED,
-        ErrorCode.RLM_ERR_SYNC_PERMISSION_DENIED -> {
+        ErrorCode.RLM_ERR_SYNC_PERMISSION_DENIED,
+        -> {
             // Permission denied errors should be unrecoverable according to Core, i.e. the
             // client will disconnect sync and transition to the "inactive" state
-            UnrecoverableSyncException(message)
+            @Suppress("DEPRECATION") io.realm.kotlin.mongodb.exceptions.UnrecoverableSyncException(
+                message
+            )
         }
+
         else -> {
             // An error happened we are not sure how to handle. Just report as a generic
             // SyncException.
             when (syncError.isFatal) {
                 false -> SyncException(message, syncError.isFatal)
-                true -> UnrecoverableSyncException(message)
+                true -> @Suppress("DEPRECATION") io.realm.kotlin.mongodb.exceptions.UnrecoverableSyncException(
+                    message
+                )
             }
         }
     }

packages/library-sync/src/commonMain/kotlin/io/realm/kotlin/mongodb/internal/SyncSessionImpl.kt

@@ -168,7 +168,7 @@ internal open class SyncSessionImpl(
             nativePointer,
             error,
             message,
-            true
+            false
         )
     }
 

packages/test-base/src/androidMain/kotlin/io/realm/kotlin/test/platform/PlatformUtils.kt

@@ -56,6 +56,10 @@ actual object PlatformUtils {
         }
         SystemClock.sleep(5000) // 5 seconds to give the GC some time to process
     }
+
+    actual fun copyFile(originPath: String, targetPath: String) {
+        File(originPath).copyTo(File(targetPath))
+    }
 }
 
 // Allocs as much garbage as we can. Pass maxSize = 0 to use all available memory in the process.

packages/test-base/src/commonMain/kotlin/io/realm/kotlin/test/platform/PlatformUtils.kt

@@ -7,6 +7,7 @@ import kotlin.time.Duration
 expect object PlatformUtils {
     fun createTempDir(prefix: String = Utils.createRandomString(16), readOnly: Boolean = false): String
     fun deleteTempDir(path: String)
+    fun copyFile(originPath: String, targetPath: String)
     fun sleep(duration: Duration)
     fun threadId(): ULong
     fun triggerGC()

packages/test-base/src/commonTest/kotlin/io/realm/kotlin/test/common/EncryptionTests.kt

@@ -69,6 +69,7 @@ class EncryptionTests {
         // Initialize an encrypted Realm
         val encryptedConf = RealmConfiguration
             .Builder(
+
                 schema = setOf(Sample::class)
             )
             .directory(tmpDir)

packages/test-base/src/jvmMain/kotlin/io/realm/kotlin/test/platform/PlatformUtils.kt

@@ -16,6 +16,7 @@
 
 package io.realm.kotlin.test.platform
 
+import java.io.File
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -41,6 +42,10 @@ actual object PlatformUtils {
         return dir.absolutePathString()
     }
 
+    actual fun copyFile(originPath: String, targetPath: String) {
+        File(originPath).copyTo(File(targetPath))
+    }
+
     actual fun deleteTempDir(path: String) {
         val rootPath: Path = Paths.get(path)
         val pathsToDelete: List<Path> =

packages/test-base/src/nativeDarwin/kotlin/io/realm/kotlin/test/platform/PlatformUtils.kt

@@ -72,4 +72,8 @@ actual object PlatformUtils {
     actual fun triggerGC() {
         GC.collect()
     }
+
+    actual fun copyFile(originPath: String, targetPath: String) {
+        platform.Foundation.NSFileManager.defaultManager.copyItemAtPath(originPath, targetPath, null)
+    }
 }

packages/test-sync/src/commonTest/kotlin/io/realm/kotlin/test/mongodb/common/AppTests.kt

@@ -438,12 +438,20 @@ class AppTests {
 
             // Create a configuration pointing to the metadata Realm for that app
             val metadataDir = "${app.configuration.syncRootDirectory}/mongodb-realm/${app.configuration.appId}/server-utility/metadata/"
+
+            // Workaround for https://github.com/realm/realm-core/issues/7876
+            // We cannot validate if the test app metadata realm is encrypted directly, as it is cached
+            // and subsequent access wont validate the encryption key. Copying the Realm allows to bypass
+            // the cache.
+            PlatformUtils.copyFile(metadataDir + "sync_metadata.realm", metadataDir + "copy_sync_metadata.realm")
+
             val wrongKey = TestHelper.getRandomKey()
             val config = RealmConfiguration
                 .Builder(setOf())
-                .name("sync_metadata.realm")
+                .name("copy_sync_metadata.realm")
                 .directory(metadataDir)
                 .encryptionKey(wrongKey)
+                .schemaVersion(7)
                 .build()
             assertTrue(fileExists(config.path))
 
@@ -478,10 +486,18 @@ class AppTests {
 
             // Create a configuration pointing to the metadata Realm for that app
             val metadataDir = "${app.configuration.syncRootDirectory}/mongodb-realm/${app.configuration.appId}/server-utility/metadata/"
+
+            // Workaround for https://github.com/realm/realm-core/issues/7876
+            // We cannot validate if the test app metadata realm is encrypted directly, as it is cached
+            // and subsequent access wont validate the encryption key. Copying the Realm allows to bypass
+            // the cache.
+            PlatformUtils.copyFile(metadataDir + "sync_metadata.realm", metadataDir + "copy_sync_metadata.realm")
+
             val config = RealmConfiguration
                 .Builder(setOf())
-                .name("sync_metadata.realm")
+                .name("copy_sync_metadata.realm")
                 .directory(metadataDir)
+                .schemaVersion(7)
                 .build()
             assertTrue(fileExists(config.path))
 

packages/test-sync/src/commonTest/kotlin/io/realm/kotlin/test/mongodb/common/FLXProgressListenerTests.kt

@@ -152,7 +152,7 @@
                 realm.syncSession.progressAsFlow(Direction.UPLOAD, ProgressMode.CURRENT_CHANGES)
                     .run {
                         withTimeout(TIMEOUT) {
                             last().let {
                                 assertTrue(it.isTransferComplete)
                                 assertEquals(1.0, it.estimate)
                             }
@@ -239,7 +239,7 @@
             try {
                 val flow = realm.syncSession.progressAsFlow(Direction.UPLOAD, ProgressMode.INDEFINITELY)
                 val job = async {
-                    withTimeout(10.seconds) {
+                    withTimeout(30.seconds) {
                         flow.collect {
                             channel.trySend(true)
                         }

packages/test-sync/src/commonTest/kotlin/io/realm/kotlin/test/mongodb/common/SyncClientResetIntegrationTests.kt

@@ -65,6 +65,7 @@ import kotlin.reflect.KClass
 import kotlin.test.AfterTest
 import kotlin.test.BeforeTest
 import kotlin.test.Test
+import kotlin.test.assertContains
 import kotlin.test.assertEquals
 import kotlin.test.assertFalse
 import kotlin.test.assertIs
@@ -710,10 +711,8 @@ class SyncClientResetIntegrationTests {
                 exception: ClientResetRequiredException
             ) {
                 // Notify that this callback has been invoked
-                assertEquals(
-                    "[Sync][AutoClientResetFailed(1028)] A fatal error occurred during client reset: 'User-provided callback failed'.",
-                    exception.message
-                )
+                assertContains(exception.message!!, "User-provided callback failed")
+
                 assertIs<IllegalStateException>(exception.cause)
                 assertEquals(
                     "User exception",
@@ -788,10 +787,8 @@ class SyncClientResetIntegrationTests {
                 exception: ClientResetRequiredException
             ) {
                 // Notify that this callback has been invoked
-                assertEquals(
-                    "[Sync][AutoClientResetFailed(1028)] A fatal error occurred during client reset: 'User-provided callback failed'.",
-                    exception.message
-                )
+                assertContains(exception.message!!, "User-provided callback failed")
+
                 channel.trySendOrFail(ClientResetEvents.ON_MANUAL_RESET_FALLBACK)
             }
         }).build()
@@ -1080,7 +1077,7 @@ class SyncClientResetIntegrationTests {
                 assertNotNull(exception.originalFilePath)
                 assertFalse(fileExists(exception.recoveryFilePath))
                 assertTrue(fileExists(exception.originalFilePath))
-                assertTrue(exception.message!!.contains("Simulate Client Reset"))
+                assertContains(exception.message!!, "Simulate Client Reset")
             }
         }
         channel.close()
@@ -1123,10 +1120,8 @@ class SyncClientResetIntegrationTests {
                 exception: ClientResetRequiredException
             ) {
                 // Notify that this callback has been invoked
-                assertEquals(
-                    "[Sync][AutoClientResetFailed(1028)] A fatal error occurred during client reset: 'User-provided callback failed'.",
-                    exception.message
-                )
+                assertContains(exception.message!!, "User-provided callback failed")
+
                 channel.trySendOrFail(ClientResetEvents.ON_MANUAL_RESET_FALLBACK)
             }
         }).build()
@@ -1193,11 +1188,8 @@ class SyncClientResetIntegrationTests {
                 // Validate that files have been moved after explicit reset
                 assertFalse(fileExists(originalFilePath))
                 assertTrue(fileExists(recoveryFilePath))
-
-                assertEquals(
-                    "[Sync][AutoClientResetFailed(1028)] A fatal error occurred during client reset: 'User-provided callback failed'.",
-                    exception.message
-                )
+                println(exception.message)
+                assertContains(exception.message!!, "User-provided callback failed")
 
                 channel.trySendOrFail(ClientResetEvents.ON_MANUAL_RESET_FALLBACK)
             }
@@ -1400,10 +1392,7 @@ class SyncClientResetIntegrationTests {
                 assertFalse(fileExists(originalFilePath))
                 assertTrue(fileExists(recoveryFilePath))
 
-                assertEquals(
-                    "[Sync][AutoClientResetFailed(1028)] A fatal error occurred during client reset: 'User-provided callback failed'.",
-                    exception.message
-                )
+                assertTrue(exception.message!!.contains("User-provided callback failed"))
 
                 channel.trySendOrFail(ClientResetEvents.ON_MANUAL_RESET_FALLBACK)
             }