Metric overhaul

CHANGELOG.md

@@ -19,7 +19,7 @@
   * AtomicFu 0.18.3 and above.
   * The new memory model only. See https://github.com/realm/realm-kotlin#kotlin-memory-model-and-coroutine-compatibility
 * Minimum Kbson 0.3.0.
-* Minimum Gradle version: 6.8.3.
+* Minimum Gradle version: 7.1.
 * Minimum Android Gradle Plugin version: 4.1.3.
 * Minimum Android SDK: 16.
 

Jenkinsfile

@@ -84,6 +84,7 @@ pipeline {
           ANDROID_NDK="${NDK_HOME}"
           ANDROID_NDK_HOME="${NDK_HOME}"
           REALM_DISABLE_ANALYTICS=true
+          REALM_PRINT_ANALYTICS=true
           JAVA_8='/Library/Java/JavaVirtualMachines/jdk1.8.0_301.jdk/Contents/Home'
           JAVA_11='/Library/Java/JavaVirtualMachines/jdk-11.0.12.jdk/Contents/Home'
           JAVA_HOME="${JAVA_11}"

examples/min-android-sample/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

packages/gradle-plugin/build.gradle.kts

@@ -93,12 +93,21 @@ sourceSets {
         java.srcDir(versionDirectory)
     }
 }
-tasks.create("pluginVersion") {
-    val outputDir = file(versionDirectory)
 
+// Task to generate gradle plugin runtime constants for SDK and core versions
+tasks.create("versionConstants") {
+    val coreDependenciesFile = layout.projectDirectory.file(
+        listOf("..", "external", "core", "dependencies.list").joinToString(File.separator)
+    )
+    val outputDir = file(versionDirectory)
     inputs.property("version", project.version)
+    inputs.file(coreDependenciesFile)
     outputs.dir(outputDir)
 
+    val versionRegex = "^VERSION=(.*)\$".toRegex()
+    val coreVersion = providers.fileContents(coreDependenciesFile).asText.get().lineSequence()
+        .map { dependecy -> versionRegex.find(dependecy)?.groups?.get(1)?.value }.single { it != null }
+
     doLast {
         val versionFile = file("$outputDir/io/realm/kotlin/gradle/version.kt")
         versionFile.parentFile.mkdirs()
@@ -107,8 +116,10 @@ tasks.create("pluginVersion") {
             // Generated file. Do not edit!
             package io.realm.kotlin.gradle
             internal const val PLUGIN_VERSION = "${project.version}"
+            internal const val CORE_VERSION = "${coreVersion}"
             """.trimIndent()
         )
     }
 }
-tasks.getByName("compileKotlin").dependsOn("pluginVersion")
+
+tasks.getByName("compileKotlin").dependsOn("versionConstants")

packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmCompilerSubplugin.kt

@@ -16,25 +16,122 @@
 
 package io.realm.kotlin.gradle
 
+import com.android.build.gradle.BaseExtension
+import io.realm.kotlin.gradle.analytics.AnalyticsService
+import io.realm.kotlin.gradle.analytics.BuilderId
+import io.realm.kotlin.gradle.analytics.ComputerId
+import io.realm.kotlin.gradle.analytics.HOST_ARCH
+import io.realm.kotlin.gradle.analytics.HOST_OS
+import io.realm.kotlin.gradle.analytics.ProjectConfiguration
+import io.realm.kotlin.gradle.analytics.TargetInfo
+import io.realm.kotlin.gradle.analytics.hexStringify
+import io.realm.kotlin.gradle.analytics.sha256Hash
+import org.gradle.api.Project
+import org.gradle.api.plugins.ExtensionAware
 import org.gradle.api.provider.Provider
+import org.gradle.api.services.BuildServiceSpec
+import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerPluginSupportPlugin
 import org.jetbrains.kotlin.gradle.plugin.SubpluginArtifact
 import org.jetbrains.kotlin.gradle.plugin.SubpluginOption
+import org.jetbrains.kotlin.gradle.plugin.cocoapods.CocoapodsExtension
+import org.jetbrains.kotlin.gradle.plugin.cocoapods.KotlinCocoapodsPlugin.Companion.COCOAPODS_EXTENSION_NAME
+import org.jetbrains.kotlin.gradle.plugin.kotlinToolingVersion
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinCommonCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmAndroidCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinSharedNativeCompilation
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaCompilation
+import org.jetbrains.kotlin.konan.target.Architecture
+import org.jetbrains.kotlin.konan.target.Family
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import java.io.File
 
 class RealmCompilerSubplugin : KotlinCompilerPluginSupportPlugin {
 
+    private lateinit var anonymizedBundleId: String
+    private var analyticsServiceProvider: Provider<AnalyticsService>? = null
+
     companion object {
-        // TODO LATER Consider embedding these from the build.gradle's pluginVersion task just
+        // TODO LATER Consider embedding these from the build.gradle's versionConstants task just
         //  as with the version. But leave it for now as they should be quite stable.
         // Modules has to match ${project.group}:${project.name} to make composite build work
         const val groupId = "io.realm.kotlin"
         const val artifactId = "plugin-compiler"
         const val version = PLUGIN_VERSION
+        const val coreVersion = CORE_VERSION
+
         // The id used for passing compiler options from command line
         const val compilerPluginId = "io.realm.kotlin"
+
         // Must match io.realm.kotlin.compiler.bundleIdKey
         const val bundleIdKey = "bundleId"
+
+        // Must match io.realm.kotlin.compiler.
+        const val featureListPathKey = "featureListPath"
+    }
+
+    @Suppress("NestedBlockDepth")
+    override fun apply(target: Project) {
+        super.apply(target)
+
+        // We build the anonymized bundle id here and pass it to the compiler plugin to ensure
+        // that the metrics and sync connection parameters are aligned.
+        val bundleId = target.rootProject.name + ":" + target.name
+        anonymizedBundleId = hexStringify(sha256Hash(bundleId.toByteArray()))
+
+        val submitAnalytics: Boolean = !target.gradle.startParameter.isOffline && !"true".equals(
+            System.getenv()["REALM_DISABLE_ANALYTICS"],
+            ignoreCase = true
+        )
+        val printAnalytics = "true".equals(System.getenv()["REALM_PRINT_ANALYTICS"], ignoreCase = true)
+        if (submitAnalytics || printAnalytics) {
+            // Identify if project is using sync by inspecting dependencies.
+            // We cannot use resolved configurations here as this code is called in
+            // afterEvaluate, and resolving it prevents other plugins from modifying
+            // them. E.g the KMP plugin will crash if we resolve the configurations
+            // in `afterEvaluate`. This means we can only see dependencies directly set,
+            // and not their transitive dependencies. This should be fine as we only
+            // want to track builds directly using Realm.
+            var usesSync = false
+            outer@
+            for (conf in target.configurations) {
+                for (dependency in conf.dependencies) {
+                    if (dependency.group == "io.realm.kotlin" && dependency.name == "library-sync") {
+                        // In Java we can detect Sync through a Gradle configuration closure.
+                        // In Kotlin, this choice is currently determined by which dependency
+                        // people include
+                        usesSync = true
+                        break@outer
+                    }
+                }
+            }
+
+            val userId = target.providers.of(ComputerId::class.java) {}.get()
+            val builderId = target.providers.of(BuilderId::class.java) {}.get()
+            val languageVersion = target.kotlinToolingVersion
+
+            analyticsServiceProvider = target.gradle.sharedServices.registerIfAbsent(
+                "Realm Analytics",
+                AnalyticsService::class.java
+            ) { spec: BuildServiceSpec<ProjectConfiguration> ->
+                spec.parameters.run {
+                    this.appId.set(anonymizedBundleId)
+                    this.userId.set(userId)
+                    this.builderId.set(builderId)
+                    this.hostOsType.set(HOST_OS.serializedName)
+                    this.hostOsVersion.set(System.getProperty("os.version"))
+                    this.hostCpuArch.set(HOST_ARCH)
+                    this.usesSync.set(usesSync)
+                    this.languageVersion.set(languageVersion.toString())
+                    this.submitAnalytics.set(submitAnalytics)
+                    this.printAnalytics.set(printAnalytics)
+                }
+            }
+        }
     }
 
     override fun isApplicable(kotlinCompilation: KotlinCompilation<*>): Boolean {
@@ -56,10 +153,138 @@ class RealmCompilerSubplugin : KotlinCompilerPluginSupportPlugin {
 
     override fun applyToCompilation(kotlinCompilation: KotlinCompilation<*>): Provider<List<SubpluginOption>> {
         val project = kotlinCompilation.target.project
-        val realmPlugin = project.plugins.getPlugin("io.realm.kotlin") as RealmPlugin
-        val anonymizedBundleId = realmPlugin.anonymizedBundleId
+
+        // Compiler plugin options
+        val options = mutableListOf(
+            SubpluginOption(key = bundleIdKey, anonymizedBundleId),
+        )
+        // Only bother collecting info if the analytics service is registered
+        analyticsServiceProvider?.let { provider ->
+            // Enable feature collection in compiler plugin by setting a path for the feature list
+            // file and pass it to the compiler plugin as a compiler plugin option
+            val featureListPath = listOf(
+                project.buildDir.path,
+                "outputs",
+                "realm-features",
+                kotlinCompilation.defaultSourceSet.name
+            ).joinToString(File.separator)
+            options.add(SubpluginOption(key = featureListPathKey, featureListPath))
+
+            // Gather target specific information
+            val targetInfo: TargetInfo? = gatherTargetInfo(kotlinCompilation)
+            // If we have something to submit register it for submission after the compilation has
+            // gathered the feature list information
+            targetInfo?.let {
+                kotlinCompilation.compileTaskProvider.get().doLast {
+                    analyticsServiceProvider!!.get().submit(targetInfo)
+                }
+            }
+        }
         return project.provider {
-            listOf(SubpluginOption(key = bundleIdKey, anonymizedBundleId))
+            options
         }
     }
 }
+
+@Suppress("ComplexMethod", "NestedBlockDepth")
+private fun gatherTargetInfo(kotlinCompilation: KotlinCompilation<*>): TargetInfo? {
+    val project = kotlinCompilation.target.project
+    return when (kotlinCompilation) {
+        // We don't send metrics for common targets but still collect features as the
+        // target specific features is a union of common and target specific features
+        is KotlinCommonCompilation,
+        is KotlinSharedNativeCompilation ->
+            null
+
+        is KotlinJvmAndroidCompilation -> {
+            val androidExtension =
+                project.extensions.findByName("android") as BaseExtension?
+            val defaultConfig = androidExtension?.defaultConfig
+            val minSDK = defaultConfig?.minSdkVersion?.apiString
+            val targetSDK = defaultConfig?.targetSdkVersion?.apiString
+            val targetCpuArch: String =
+                defaultConfig?.ndk?.abiFilters?.singleOrNull()?.let { androidArch(it) }
+                    ?: "Universal"
+            TargetInfo("Android", targetCpuArch, targetSDK, minSDK)
+        }
+
+        is KotlinJvmCompilation -> {
+            val jvmTarget = kotlinCompilation.kotlinOptions.jvmTarget
+            TargetInfo("JVM", "Universal", jvmTarget, jvmTarget)
+        }
+
+        is KotlinNativeCompilation -> {
+            // We currently only support Darwin targets, so assume that we can pull minSdk
+            // from the given deploymentTarget. Non-CocoaPod Xcode project have this in its
+            // pdxproj-file as <X>OS_DEPLOYMENT_TARGET, but assuming that most people use
+            // CocoaPods as it is the default. Reevaluate if we see too many missing values.
+            val kotlinExtension: KotlinMultiplatformExtension =
+                project.extensions.getByType(KotlinMultiplatformExtension::class.java)
+            val cocoapodsExtension =
+                (kotlinExtension as ExtensionAware).extensions.findByName(
+                    COCOAPODS_EXTENSION_NAME
+                ) as CocoapodsExtension?
+            val minSdk = cocoapodsExtension?.let { cocoapods ->
+                when (kotlinCompilation.konanTarget.family) {
+                    Family.OSX -> cocoapods.osx.deploymentTarget
+                    Family.IOS -> cocoapods.ios.deploymentTarget
+                    Family.TVOS -> cocoapods.tvos.deploymentTarget
+                    Family.WATCHOS -> cocoapods.watchos.deploymentTarget
+                    Family.LINUX,
+                    Family.MINGW,
+                    Family.ANDROID,
+                    Family.WASM,
+                    Family.ZEPHYR -> null // Not supported yet
+                }
+            }
+            TargetInfo(
+                nativeTarget(kotlinCompilation.konanTarget),
+                nativeArch(kotlinCompilation.konanTarget),
+                null,
+                minSdk
+            )
+        }
+        // Not supported yet so don't try to gather target information
+        is KotlinJsCompilation,
+        is KotlinWithJavaCompilation<*, *> -> null
+
+        else -> {
+            null
+        }
+    }
+}
+
+// Helper method to ensure that we align target type string for native builds
+fun nativeTarget(target: KonanTarget) = when (target.family) {
+    Family.OSX -> "macOS"
+    Family.IOS -> "iOS"
+    Family.TVOS -> "tvOS"
+    Family.WATCHOS -> "watchOS"
+    Family.LINUX -> "Linux"
+    Family.MINGW -> "MinGW"
+    Family.ANDROID -> "Android(native)"
+    Family.WASM -> "Wasm"
+    Family.ZEPHYR -> "Zephyr"
+    else -> "Unknown[${target.family}]"
+}
+
+// Helper method to ensure that we align architecture strings for Kotlin native builds
+fun nativeArch(target: KonanTarget) = when (target.architecture) {
+    Architecture.X64 -> io.realm.kotlin.gradle.analytics.Architecture.X64.serializedName
+    Architecture.X86 -> io.realm.kotlin.gradle.analytics.Architecture.X86.serializedName
+    Architecture.ARM64 -> io.realm.kotlin.gradle.analytics.Architecture.ARM64.serializedName
+    Architecture.ARM32 -> io.realm.kotlin.gradle.analytics.Architecture.ARM.serializedName
+    Architecture.MIPS32 -> "Mips"
+    Architecture.MIPSEL32 -> "MipsEL32"
+    Architecture.WASM32 -> "Wasm"
+    else -> "Unknown[${target.architecture}]"
+}
+
+// Helper method to ensure that we align architecture strings for Android platforms
+fun androidArch(target: String): String = when (target) {
+    "armeabi-v7a" -> io.realm.kotlin.gradle.analytics.Architecture.ARM.serializedName
+    "arm64-v8a" -> io.realm.kotlin.gradle.analytics.Architecture.ARM64.serializedName
+    "x86" -> io.realm.kotlin.gradle.analytics.Architecture.X86.serializedName
+    "x86_64" -> io.realm.kotlin.gradle.analytics.Architecture.X64.serializedName
+    else -> "Unknown[$target]"
+}

packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmPlugin.kt

@@ -16,44 +16,20 @@
 
 package io.realm.kotlin.gradle
 
-import io.realm.kotlin.gradle.analytics.AnalyticsService
-import io.realm.kotlin.gradle.analytics.hexStringify
-import io.realm.kotlin.gradle.analytics.sha256Hash
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.DependencySubstitutions
 import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
-import org.gradle.api.provider.Provider
-import org.gradle.build.event.BuildEventsListenerRegistry
-import javax.inject.Inject
 
 @Suppress("unused")
 open class RealmPlugin : Plugin<Project> {
 
     private val logger: Logger = Logging.getLogger("realm-plugin")
 
-    internal lateinit var anonymizedBundleId: String
-
-    @Inject
-    public open fun getBuildEventsRegistry(): BuildEventsListenerRegistry { TODO("Should have been replaced by Gradle.") }
-
     override fun apply(project: Project) {
         project.pluginManager.apply(RealmCompilerSubplugin::class.java)
-
-        // We build the anonymized bundle id here and pass it to the compiler plugin to ensure
-        // that the metrics and sync connection parameters are aligned.
-        val bundleId = project.rootProject.name + ":" + project.name
-        anonymizedBundleId = hexStringify(sha256Hash(bundleId.toByteArray()))
-
-        // Run analytics as a Build Service to support Gradle Configuration Cache
-        val serviceProvider: Provider<AnalyticsService> = project.gradle.sharedServices.registerIfAbsent(
-            "realm-analytics",
-            AnalyticsService::class.java
-        ) { /* Do nothing */ }
-        getBuildEventsRegistry().onTaskCompletion(serviceProvider)
-
         project.configurations.all { conf: Configuration ->
             // Ensure that android unit tests uses the Realm JVM variant rather than Android.
             // This is a bit britle. See https://github.com/realm/realm-kotlin/issues/1404 for
@@ -71,28 +47,5 @@ open class RealmPlugin : Plugin<Project> {
                 }
             }
         }
-
-        // Stand alone Android projects have not initialized kotlin plugin when applying this, so
-        // postpone dependency injection till after evaluation.
-        project.afterEvaluate {
-            // TODO AUTO-SETUP To ease configuration we could/should inject dependencies to our
-            //  library, but await better insight into when/what to inject and supply appropriate
-            //  opt-out options through our own extension?
-            //  Dependencies should probably be added by source set and not by target, as
-            //  kotlin.sourceSets.getByName("commonMain").dependencies (or "main" for Android), but
-
-            // Create the analytics during configuration because it needs access to the project
-            // in order to gather project relevant information in afterEvaluate. Currently
-            // there doesn't seem a way to get this information during the Execution Phase.
-            @Suppress("SwallowedException", "TooGenericExceptionCaught")
-            try {
-                val analyticsService: AnalyticsService = serviceProvider.get()
-                analyticsService.collectAnalyticsData(it)
-            } catch (ex: Exception) {
-                // Work-around for https://github.com/gradle/gradle/issues/18821
-                // Since this only happens in multi-module projects, this should be fine as
-                // the build will still be registered by the first module that starts the service.
-            }
-        }
     }
 }