From 64d23c6b88a68b5dfc65f5bf4749701423e6ced3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20R=C3=B8rbech?= Date: Wed, 6 Sep 2023 17:22:31 +0200 Subject: [PATCH] Metric overhaul --- packages/gradle-plugin/build.gradle.kts | 26 +- .../kotlin/gradle/RealmCompilerSubplugin.kt | 180 +++++++- .../io/realm/kotlin/gradle/RealmPlugin.kt | 76 ---- .../gradle/analytics/AnalyticsService.kt | 394 +++++++++++++++--- .../kotlin/gradle/analytics/RealmAnalytics.kt | 329 --------------- .../compiler/RealmCommandLineProcessor.kt | 17 +- 6 files changed, 558 insertions(+), 464 deletions(-) delete mode 100644 packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/RealmAnalytics.kt diff --git a/packages/gradle-plugin/build.gradle.kts b/packages/gradle-plugin/build.gradle.kts index da97942a8e..d917371c9d 100644 --- a/packages/gradle-plugin/build.gradle.kts +++ b/packages/gradle-plugin/build.gradle.kts @@ -14,6 +14,7 @@ * limitations under the License. */ import kotlin.text.toBoolean +import java.io.ByteArrayOutputStream plugins { kotlin("jvm") @@ -93,22 +94,41 @@ sourceSets { java.srcDir(versionDirectory) } } -tasks.create("pluginVersion") { - val outputDir = file(versionDirectory) +// Task to fetch core version from dependency.list from core submodule +tasks.create("coreVersion", Exec::class.java) { + workingDir = File(listOf("..", "external", "core").joinToString(File.separator)) + commandLine = listOf("grep", "^VERSION", "dependencies.list") + standardOutput = ByteArrayOutputStream() + doLast { + extra["output"] = try { + standardOutput.toString().trim().split("=")[1] + } catch (e: Exception) { + "" + } + } +} + +// Task to generate gradle plugin runtime constants for SDK and core versions +tasks.create("versionConstants") { + val outputDir = file(versionDirectory) inputs.property("version", project.version) outputs.dir(outputDir) + dependsOn(tasks.get("coreVersion")) doLast { val versionFile = file("$outputDir/io/realm/kotlin/gradle/version.kt") + val coreVersion = (tasks.get("coreVersion") as Exec).extra["output"] versionFile.parentFile.mkdirs() versionFile.writeText( """ // Generated file. Do not edit! package io.realm.kotlin.gradle internal const val PLUGIN_VERSION = "${project.version}" + internal const val CORE_VERSION = "${coreVersion}" """.trimIndent() ) } } -tasks.getByName("compileKotlin").dependsOn("pluginVersion") + +tasks.getByName("compileKotlin").dependsOn("versionConstants") diff --git a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmCompilerSubplugin.kt b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmCompilerSubplugin.kt index 906e853c75..cbc762d712 100644 --- a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmCompilerSubplugin.kt +++ b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmCompilerSubplugin.kt @@ -16,25 +16,95 @@ package io.realm.kotlin.gradle +import com.android.build.gradle.BaseExtension +import io.realm.kotlin.gradle.analytics.AnalyticsService +import io.realm.kotlin.gradle.analytics.TargetInfo +import io.realm.kotlin.gradle.analytics.hexStringify +import io.realm.kotlin.gradle.analytics.sha256Hash +import org.gradle.api.Project +import org.gradle.api.plugins.ExtensionAware import org.gradle.api.provider.Provider +import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation import org.jetbrains.kotlin.gradle.plugin.KotlinCompilerPluginSupportPlugin import org.jetbrains.kotlin.gradle.plugin.SubpluginArtifact import org.jetbrains.kotlin.gradle.plugin.SubpluginOption +import org.jetbrains.kotlin.gradle.plugin.cocoapods.CocoapodsExtension +import org.jetbrains.kotlin.gradle.plugin.cocoapods.KotlinCocoapodsPlugin.Companion.COCOAPODS_EXTENSION_NAME +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinCommonCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmAndroidCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinSharedNativeCompilation +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaCompilation +import org.jetbrains.kotlin.konan.target.Architecture +import org.jetbrains.kotlin.konan.target.Family +import org.jetbrains.kotlin.konan.target.KonanTarget +import java.io.File class RealmCompilerSubplugin : KotlinCompilerPluginSupportPlugin { + private lateinit var anonymizedBundleId: String + private var analyticsServiceProvider: Provider? = null + companion object { - // TODO LATER Consider embedding these from the build.gradle's pluginVersion task just + // TODO LATER Consider embedding these from the build.gradle's versionConstants task just // as with the version. But leave it for now as they should be quite stable. // Modules has to match ${project.group}:${project.name} to make composite build work const val groupId = "io.realm.kotlin" const val artifactId = "plugin-compiler" const val version = PLUGIN_VERSION + const val coreVersion = CORE_VERSION + // The id used for passing compiler options from command line const val compilerPluginId = "io.realm.kotlin" + // Must match io.realm.kotlin.compiler.bundleIdKey const val bundleIdKey = "bundleId" + // Must match io.realm.kotlin.compiler. + const val featureListPathKey = "featureListPath" + } + + override fun apply(project: Project) { + super.apply(project) + + // We build the anonymized bundle id here and pass it to the compiler plugin to ensure + // that the metrics and sync connection parameters are aligned. + val bundleId = project.rootProject.name + ":" + project.name + anonymizedBundleId = hexStringify(sha256Hash(bundleId.toByteArray())) + + val disableAnalytics: Boolean = project.gradle.startParameter.isOffline || "true".equals( + System.getenv()["REALM_DISABLE_ANALYTICS"], + ignoreCase = true + ) + if (!disableAnalytics) { + // Indentify if project is using sync by inspecting dependencies. + // We cannot use resolved configurations here as this code is called in + // afterEvaluate, and resolving it prevents other plugins from modifying + // them. E.g the KMP plugin will crash if we resolve the configurations + // in `afterEvaluate`. This means we can only see dependencies directly set, + // and not their transitive dependencies. This should be fine as we only + // want to track builds directly using Realm. + var usesSync = false + outer@ + for (conf in project.configurations) { + for (dependency in conf.dependencies) { + if (dependency.group == "io.realm.kotlin" && dependency.name == "library-sync") { + // In Java we can detect Sync through a Gradle configuration closure. + // In Kotlin, this choice is currently determined by which dependency + // people include + usesSync = true + break@outer + } + } + } + analyticsServiceProvider = project.gradle.sharedServices.registerIfAbsent( + "Realm Analytics", + AnalyticsService::class.java + ) {} + analyticsServiceProvider!!.get().init(anonymizedBundleId, usesSync) + } } override fun isApplicable(kotlinCompilation: KotlinCompilation<*>): Boolean { @@ -56,10 +126,112 @@ class RealmCompilerSubplugin : KotlinCompilerPluginSupportPlugin { override fun applyToCompilation(kotlinCompilation: KotlinCompilation<*>): Provider> { val project = kotlinCompilation.target.project - val realmPlugin = project.plugins.getPlugin("io.realm.kotlin") as RealmPlugin - val anonymizedBundleId = realmPlugin.anonymizedBundleId + + // Compiler plugin options + val options = mutableListOf( + SubpluginOption(key = bundleIdKey, anonymizedBundleId), + ) + // Only bother collecting info if the analytics service is registered + analyticsServiceProvider?.let { provider -> + // Enable feature collection in compiler plugin by setting a path for the feature set file + val featureListPath = listOf( project.buildDir.path, "outputs", "realm-features", kotlinCompilation.defaultSourceSet.name ).joinToString( File.separator ) + options.add(SubpluginOption(key = featureListPathKey, featureListPath)) + + val targetInfo: TargetInfo? = when (kotlinCompilation) { + // We don't send metrics for common targets but still collect features as the + // target specific features is a union of common and target specific features + is KotlinCommonCompilation, + is KotlinSharedNativeCompilation -> + null + is KotlinJvmAndroidCompilation -> { + val androidExtension = + project.extensions.findByName("android") as BaseExtension? + val defaultConfig = androidExtension?.defaultConfig + val minSDK = defaultConfig?.minSdkVersion?.apiString + val targetSDK = defaultConfig?.targetSdkVersion?.apiString + val targetCpuArch: String = + defaultConfig?.ndk?.abiFilters?.singleOrNull()?.let { androidArch(it) } + ?: "Universal" + TargetInfo("Android", targetCpuArch, targetSDK, minSDK) + } + is KotlinJvmCompilation -> { + val jvmTarget = kotlinCompilation.kotlinOptions.jvmTarget + TargetInfo("JVM", "Universal", jvmTarget, jvmTarget) + } + is KotlinNativeCompilation -> { + // We currently only support Darwin targets, so assume that we can pull minSdk + // from the given deploymentTarget. Non-CocoaPod Xcode project have this in its + // pdxproj-file as OS_DEPLOYMENT_TARGET, but assuming that most people use + // CocoaPods as it is the default. Reevaluate if we see too many missing values. + val kotlinExtension: KotlinMultiplatformExtension = project.extensions.getByType(KotlinMultiplatformExtension::class.java) + val cocoapodsExtension = (kotlinExtension as ExtensionAware).extensions.getByName(COCOAPODS_EXTENSION_NAME) as CocoapodsExtension? + val minSdk = cocoapodsExtension?.let { cocoapods -> + when (kotlinCompilation.konanTarget.family) { + Family.OSX -> cocoapods.osx.deploymentTarget + Family.IOS -> cocoapods.ios.deploymentTarget + Family.TVOS -> cocoapods.tvos.deploymentTarget + Family.WATCHOS -> cocoapods.watchos.deploymentTarget + Family.LINUX, + Family.MINGW, + Family.ANDROID, + Family.WASM, + Family.ZEPHYR -> null // Not supported yet + } + } + TargetInfo(nativeTarget(kotlinCompilation.konanTarget), nativeArch(kotlinCompilation.konanTarget), null, minSdk) + } + // Not supported yet so don't try to gather target information + is KotlinJsCompilation, + is KotlinWithJavaCompilation<*, *> -> null + else -> { + null + } + } + // If we have something to submit register it for submission after the compilation has + // gathered feature information + targetInfo?.let { + kotlinCompilation.compileTaskProvider.get().doLast { + analyticsServiceProvider!!.get().submit(targetInfo) + } + } + } return project.provider { - listOf(SubpluginOption(key = bundleIdKey, anonymizedBundleId)) + options } } } + +// Helper method to ensure that we align target type string for native builds +fun nativeTarget(target: KonanTarget) = when (target.family) { + Family.OSX -> "macOS" + Family.IOS -> "iOS" + Family.TVOS -> "tvOS" + Family.WATCHOS -> "watchOS" + Family.LINUX -> "Linux" + Family.MINGW -> "MinGW" + Family.ANDROID -> "Android(native)" + Family.WASM -> "Wasm" + Family.ZEPHYR -> "Zephyr" + else -> "Unknown[${target.family}]" +} + +// Helper method to ensure that we align architecture strings for Kotlin native builds +fun nativeArch(target: KonanTarget) = when (target.architecture) { + Architecture.X64 -> io.realm.kotlin.gradle.analytics.Architecture.X64.serializedName + Architecture.X86 -> io.realm.kotlin.gradle.analytics.Architecture.X86.serializedName + Architecture.ARM64 -> io.realm.kotlin.gradle.analytics.Architecture.ARM64.serializedName + Architecture.ARM32 -> io.realm.kotlin.gradle.analytics.Architecture.ARM.serializedName + Architecture.MIPS32 -> "Mips" + Architecture.MIPSEL32 -> "MipsEL32" + Architecture.WASM32 -> "Wasm" + else -> "Unknown[${target.architecture}]" +} + +// Helper method to ensure that we align architecture strings for Android platforms +fun androidArch(target: String): String = when (target) { + "armeabi-v7a" -> io.realm.kotlin.gradle.analytics.Architecture.ARM.serializedName + "arm64-v8a" -> io.realm.kotlin.gradle.analytics.Architecture.ARM64.serializedName + "x86" -> io.realm.kotlin.gradle.analytics.Architecture.X86.serializedName + "x86_64" -> io.realm.kotlin.gradle.analytics.Architecture.X64.serializedName + else -> "Unknown[${target}]" +} diff --git a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmPlugin.kt b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmPlugin.kt index 00a2225112..22e138ee1e 100644 --- a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmPlugin.kt +++ b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/RealmPlugin.kt @@ -16,48 +16,20 @@ package io.realm.kotlin.gradle -import io.realm.kotlin.gradle.analytics.AnalyticsService -import io.realm.kotlin.gradle.analytics.hexStringify -import io.realm.kotlin.gradle.analytics.sha256Hash import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.DependencySubstitutions import org.gradle.api.logging.Logger import org.gradle.api.logging.Logging -import org.gradle.api.provider.Provider -import org.gradle.build.event.BuildEventsListenerRegistry -import org.jetbrains.kotlin.gradle.dsl.KotlinJvmOptions -import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension -import org.jetbrains.kotlin.gradle.dsl.KotlinSingleTargetExtension -import org.jetbrains.kotlin.gradle.plugin.KotlinTarget -import javax.inject.Inject @Suppress("unused") open class RealmPlugin : Plugin { private val logger: Logger = Logging.getLogger("realm-plugin") - internal lateinit var anonymizedBundleId: String - - @Inject - public open fun getBuildEventsRegistry(): BuildEventsListenerRegistry { TODO("Should have been replaced by Gradle.") } - override fun apply(project: Project) { project.pluginManager.apply(RealmCompilerSubplugin::class.java) - - // We build the anonymized bundle id here and pass it to the compiler plugin to ensure - // that the metrics and sync connection parameters are aligned. - val bundleId = project.rootProject.name + ":" + project.name - anonymizedBundleId = hexStringify(sha256Hash(bundleId.toByteArray())) - - // Run analytics as a Build Service to support Gradle Configuration Cache - val serviceProvider: Provider = project.gradle.sharedServices.registerIfAbsent( - "realm-analytics", - AnalyticsService::class.java - ) { /* Do nothing */ } - getBuildEventsRegistry().onTaskCompletion(serviceProvider) - project.configurations.all { conf: Configuration -> // Ensure that android unit tests uses the Realm JVM variant rather than Android. // This is a bit britle. See https://github.com/realm/realm-kotlin/issues/1404 for @@ -75,53 +47,5 @@ open class RealmPlugin : Plugin { } } } - - // Stand alone Android projects have not initialized kotlin plugin when applying this, so - // postpone dependency injection till after evaluation. - project.afterEvaluate { - val kotlin: Any? = project.extensions.findByName("kotlin") - // TODO AUTO-SETUP To ease configuration we could/should inject dependencies to our - // library, but await better insight into when/what to inject and supply appropriate - // opt-out options through our own extension? - // Dependencies should probably be added by source set and not by target, as - // kotlin.sourceSets.getByName("commonMain").dependencies (or "main" for Android), but - when (kotlin) { - is KotlinSingleTargetExtension<*> -> { - updateKotlinOption(kotlin.target) - } - is KotlinMultiplatformExtension -> { - kotlin.targets.all { target -> updateKotlinOption(target) } - } - else -> { - // TODO AUTO-SETUP Should we report errors? Probably an oversighted case - // TODO("Cannot 'realm-kotlin' library dependency to ${if (kotlin != null) kotlin::class.qualifiedName else "null"}") - } - } - - // Create the analytics during configuration because it needs access to the project - // in order to gather project relevant information in afterEvaluate. Currently - // there doesn't seem a way to get this information during the Execution Phase. - @Suppress("SwallowedException", "TooGenericExceptionCaught") - try { - val analyticsService: AnalyticsService = serviceProvider.get() - analyticsService.collectAnalyticsData(it) - } catch (ex: Exception) { - // Work-around for https://github.com/gradle/gradle/issues/18821 - // Since this only happens in multi-module projects, this should be fine as - // the build will still be registered by the first module that starts the service. - } - } - } - - private fun updateKotlinOption(target: KotlinTarget) { - target.compilations.all { compilation -> - // Setup correct compiler options - // FIXME AUTO-SETUP Are these to dangerous to apply under the hood? - when (val options = compilation.kotlinOptions) { - is KotlinJvmOptions -> { - options.jvmTarget = "1.8" - } - } - } } } diff --git a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/AnalyticsService.kt b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/AnalyticsService.kt index 50ff1d778c..db40c1c387 100644 --- a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/AnalyticsService.kt +++ b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/AnalyticsService.kt @@ -15,18 +15,24 @@ */ package io.realm.kotlin.gradle.analytics -import org.gradle.api.Project +import io.realm.kotlin.gradle.RealmCompilerSubplugin import org.gradle.api.logging.Logger import org.gradle.api.logging.Logging import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters -import org.gradle.tooling.events.FinishEvent -import org.gradle.tooling.events.OperationCompletionListener -import org.gradle.tooling.events.task.TaskFailureResult -import org.gradle.tooling.events.task.TaskFinishEvent -import org.gradle.tooling.events.task.TaskOperationResult -import org.gradle.tooling.events.task.TaskSkippedResult -import org.gradle.tooling.events.task.TaskSuccessResult +import java.io.File +import java.io.FileNotFoundException +import java.io.IOException +import java.io.UnsupportedEncodingException +import java.net.HttpURLConnection +import java.net.NetworkInterface +import java.net.SocketException +import java.net.URL +import java.security.MessageDigest +import java.security.NoSuchAlgorithmException +import java.util.Scanner +import javax.xml.bind.DatatypeConverter +import kotlin.experimental.and /** * Analytics Build Service responsible for triggering analytics at the correct time. @@ -39,63 +45,353 @@ import org.gradle.tooling.events.task.TaskSuccessResult * * **See:** [Build Services](https://docs.gradle.org/current/userguide/build_services.html) */ -abstract class AnalyticsService : BuildService, OperationCompletionListener { - private val logger: Logger = Logging.getLogger("realm-build-service") - private var analytics: RealmAnalytics? = null +// Asynchronously submits build information to Realm when the gradle compile task run +// +// To be clear: this does *not* run when your app is in production or on +// your end-user's devices; it will only run when you build your app from source. +// +// Why are we doing this? Because it helps us build a better product for you. +// None of the data personally identifies you, your employer or your app, but it +// *will* help us understand what Realm version you use, what host OS you use, +// etc. Having this info will help with prioritizing our time, adding new +// features and deprecating old features. Collecting an anonymized bundle & +// anonymized MAC is the only way for us to count actual usage of the other +// metrics accurately. If we don't have a way to deduplicate the info reported, +// it will be useless, as a single developer building their app on Windows ten +// times would report 10 times more than a single developer that only builds +// once from Mac OS X, making the data all but useless. No one likes sharing +// data unless it's necessary, we get it, and we've debated adding this for a +// long long time. Since Realm is a free product without an email signup, we +// feel this is a necessary step so we can collect relevant data to build a +// better product for you. +// +// Currently the following information is reported: +// - What version of Realm is being used +// - What OS you are running on +// - An anonymized MAC address and bundle ID to aggregate the other information on. +// +// The collected information can be inspected by settings the system environment variable +// REALM_PRINT_ANALYTICS=true +// Collection and submission of data can be fully disabled by setting the system environment variable +// REALM_DISABLE_ANALYTICS=true - /** - * Only lifecycle event currently available in Build Services. - */ - override fun onFinish(event: FinishEvent?) { - @Suppress("TooGenericExceptionCaught") +private const val TOKEN = "ce0fac19508f6c8f20066d345d360fd0" +private const val EVENT_NAME = "Run" +private const val URL_PREFIX = "https://data.mongodb-api.com/app/realmsdkmetrics-zmhtm/endpoint/metric_webhook/metric?data=" + +// Container object for project specific details, thus equal across all platforms. +data class ProjectInfo( + val appId: String, + val userId: String, + val builderId: String, + val hostOsType: String, + val hostOsVersion: String, + val hostCpuArch: String, + val usesSync: Boolean, +); + +// Container object for target specific details. +data class TargetInfo( + val targetOsType: String, + val targetCpuArch: String, + val targetOSVersion: String?, + val targetOSMinVersion: String?, +) + +abstract class AnalyticsService : BuildService { + private val logger: Logger = Logging.getLogger("realm-analytics") + + private lateinit var projectInfo: ProjectInfo + private var verbose: Boolean = false + + fun init(anonymizedBundleId: String, usesSync: Boolean) { + projectInfo = ProjectInfo( + appId = anonymizedBundleId, + userId = ComputerIdentifierGenerator.get(), + builderId = BuilderIdentifierGenerator.get(), + hostOsType = HOST_OS.serializedName, + hostOsVersion = System.getProperty("os.version"), + hostCpuArch = HOST_ARCH, + usesSync = usesSync, + ) + verbose = "true".equals(System.getenv()["REALM_PRINT_ANALYTICS"], ignoreCase = true) + } + + fun submit(targetInfo: TargetInfo) { + val json = """ + { + "event": "$EVENT_NAME", + "properties": { + "token": "$TOKEN", + "distinct_id": "${projectInfo.userId}", + "builder_id: "${projectInfo.builderId}", + "Anonymized MAC Address": "${projectInfo.userId}", + "Anonymized Bundle ID": "${projectInfo.appId}", + "Binding": "kotlin", + "Language": "kotlin", + "Host OS Type": "${projectInfo.hostOsType}", + "Host OS Version": "${projectInfo.hostOsVersion}", + "Host CPU Arch": "${projectInfo.hostCpuArch}", + "Target CPU Arch": "${targetInfo.targetCpuArch}", + "Target OS Type": "${targetInfo.targetOsType}", + "Target OS Minimum Version": "${targetInfo.targetOSMinVersion}", + "Target OS Version": "${targetInfo.targetOSVersion}" + "Realm Version": "${RealmCompilerSubplugin.version}", + "Core Version": "${RealmCompilerSubplugin.coreVersion}", + "Sync Enabled": ${if (projectInfo.usesSync) "true" else "false"}, + } + }""".trimIndent() + sendAnalytics(json) + } + + @Suppress("TooGenericExceptionCaught") + private fun sendAnalytics(json: String) { try { - if (event == null) { - logger.warn("Null event received. This should never happen.") - return + if (!verbose) { + debug("Submitting analytics payload:\n$json") + } else { + warn("Submitting analytics payload:\n$json") } - when (event) { - is TaskFinishEvent -> handleTaskResult(event) + Thread { + try { + val response = networkQuery(json) + debug("Analytics payload sent: $response") + } catch (e: InterruptedException) { + debug("Submitting analytics was interrupted.") + } + }.apply { + isDaemon = true + }.start() + } catch (e: Exception) { + // Analytics failing for any reason should not crash the build + debug("Submitting analytics payload failed: $e") + } + } + + private fun networkQuery(jsonPayload: String): Int { + try { + val url = URL(URL_PREFIX + base64Encode(jsonPayload)) + val connection = url.openConnection() as HttpURLConnection + connection.requestMethod = "GET" + connection.connect() + return connection.responseCode + } catch (ignored: Throwable) { + return -1 + } + } + + private fun tag(message: String): String = "[REALM-ANALYTICS] $message" + private fun debug(message: String) = logger.debug(tag(message)) + private fun warn(message: String) = logger.warn(tag(message)) +} + +/** + * Generate a unique identifier for a computer. The method being used depends on the platform: + * - OS X: Mac address of en0 + * - Windows: BIOS identifier + * - Linux: Machine ID provided by the OS + */ +internal object ComputerIdentifierGenerator { + private const val UNKNOWN = "unknown" + private val OS = System.getProperty("os.name").lowercase() + @Suppress("TooGenericExceptionCaught", "SwallowedException") + fun get(): String { + return try { + when { + isWindows -> { + windowsIdentifier + } + isMac -> { + macOsIdentifier + } + isLinux -> { + linuxMacAddress + } else -> { - logger.warn("Unknown event type: ${event.javaClass.name}") + UNKNOWN } } - } catch (ex: Exception) { - logger.warn("Unexpected error: $ex") + } catch (e: Exception) { + UNKNOWN } } - private fun handleTaskResult(taskEvent: TaskFinishEvent) { - when (val result: TaskOperationResult = taskEvent.result) { - is TaskSkippedResult -> { /* Ignore skipped tasks to avoid excessive work during incremental builds */ } - is TaskFailureResult -> { filterResultAndSendAnalytics(taskEvent) } - is TaskSuccessResult -> { filterResultAndSendAnalytics(taskEvent) } - else -> { - logger.warn("Unknown task type: ${result.javaClass.name}") + private val isWindows: Boolean + get() = OS.contains("win") + private val isMac: Boolean + get() = OS.contains("mac") + private val isLinux: Boolean + get() = OS.contains("inux") + + @get:Throws(FileNotFoundException::class, NoSuchAlgorithmException::class) + private val linuxMacAddress: String + get() { + var machineId = File("/var/lib/dbus/machine-id") + if (!machineId.exists()) { + machineId = File("/etc/machine-id") + } + if (!machineId.exists()) { + return UNKNOWN + } + var scanner: Scanner? = null + return try { + scanner = Scanner(machineId) + val id = scanner.useDelimiter("\\A").next() + hexStringify(sha256Hash(id.toByteArray())) + } finally { + scanner?.close() + } + } + + @get:Throws(SocketException::class, NoSuchAlgorithmException::class) + private val macOsIdentifier: String + get() { + val networkInterface = NetworkInterface.getByName("en0") + val hardwareAddress = networkInterface.hardwareAddress + return hexStringify(sha256Hash(hardwareAddress)) + } + + @get:Throws(IOException::class, NoSuchAlgorithmException::class) + private val windowsIdentifier: String + get() { + val runtime = Runtime.getRuntime() + val process = runtime.exec(arrayOf("wmic", "csproduct", "get", "UUID")) + var result: String? = null + val `is` = process.inputStream + val sc = Scanner(process.inputStream) + `is`.use { + while (sc.hasNext()) { + val next = sc.next() + if (next.contains("UUID")) { + result = sc.next().trim { it <= ' ' } + break + } + } + } + return if (result == null) UNKNOWN else hexStringify(sha256Hash(result!!.toByteArray())) + } +} +internal object BuilderIdentifierGenerator { + private const val UNKNOWN = "unknown" + @Suppress("TooGenericExceptionCaught", "SwallowedException") + fun get(): String { + return try { + val hostIdentifier = when (HOST_OS){ + Host.WINDOWS -> windowsIdentifier + Host.MACOS -> macOsIdentifier + Host.LINUX -> linuxMacAddress + else -> throw RuntimeException("Unkown host identifier") } + val data = "Realm is great" + hostIdentifier + return base64Encode(sha256Hash(data.toByteArray()))!! + } catch (e: Exception) { + UNKNOWN } } - private fun filterResultAndSendAnalytics(taskEvent: TaskFinishEvent) { - // We use `compile` tasks as a heuristic for a "build". This will not detect builds - // that fail very early or incremental builds with no code change, but neither will it - // trigger for tasks unrelated to building code. A normal build consists of multiple - // compile tasks, but the RealmAnalytics class tracks this and only send analytics once. - if (taskEvent.descriptor.name.contains("compile", true)) { - analytics?.sendAnalyticsData() + @get:Throws(FileNotFoundException::class, NoSuchAlgorithmException::class) + private val linuxMacAddress: String + get() { + return File("/etc/machine-id").inputStream().readBytes().toString().trim() + } + + @get:Throws(SocketException::class, NoSuchAlgorithmException::class) + private val macOsIdentifier: String + get() { + val runtime = Runtime.getRuntime() + val process = runtime.exec(arrayOf("ioreg", "-rd1", "-c", "IOPlatformExpertDevice")) + val regEx = ".*\"IOPlatformUUID\"\\s=\\s\"(.+)\"".toRegex() + val input = String(process.inputStream.readBytes()) + val find: MatchResult? = regEx.find(input) + return find?.groups?.get(1)?.value!! + } + + @get:Throws(IOException::class, NoSuchAlgorithmException::class) + private val windowsIdentifier: String + get() { + val runtime = Runtime.getRuntime() + val process = runtime.exec(arrayOf("Reg", "QUERY", "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Cryptography", "MachineGuid")) + val input = String(process.inputStream.readAllBytes()) + // Manually expanded [:alnum:] as ([[:alnum:]-]+) didn't seems to work + val regEx = "\\s*MachineGuid\\s*\\w*\\s*([A-Za-z0-9-]+)".toRegex() + val find: MatchResult? = regEx.find(input) + return find?.groups?.get(1)?.value!! } +} + +/** + * Encode the given string with Base64 + * @param data the string to encode + * @return the encoded string + * @throws UnsupportedEncodingException + */ +@Throws(UnsupportedEncodingException::class) +internal fun base64Encode(data: String): String? { + return base64Encode(data.toByteArray(charset("UTF-8"))) +} + +internal fun base64Encode(data: ByteArray): String? { + return DatatypeConverter.printBase64Binary(data) +} + +/** + * Compute the SHA-256 hash of the given byte array + * @param data the byte array to hash + * @return the hashed byte array + * @throws NoSuchAlgorithmException + */ +@Throws(NoSuchAlgorithmException::class) +internal fun sha256Hash(data: ByteArray?): ByteArray { + val messageDigest = MessageDigest.getInstance("SHA-256") + return messageDigest.digest(data) +} + +/** + * Convert a byte array to its hex-string + * @param data the byte array to convert + * @return the hex-string of the byte array + */ +@Suppress("MagicNumber") +internal fun hexStringify(data: ByteArray): String { + val stringBuilder = java.lang.StringBuilder() + for (singleByte: Byte in data) { + stringBuilder.append(((singleByte and 0xff.toByte()) + 0x100).toString(16).substring(1)) + } + return stringBuilder.toString() +} + +enum class Host(val serializedName: String) { + WINDOWS("Windows"), LINUX("Linux"), MACOS("macOs"), UNKNOWN("Unknown"); +} + +/** + * Define which Host OS the build is running on. + */ +val HOST_OS: Host = run { + val hostOs = System.getProperty("os.name") + when { + hostOs.contains("windows", ignoreCase = true) -> Host.WINDOWS + hostOs.contains("inux", ignoreCase = true) -> Host.LINUX + hostOs.contains("mac", ignoreCase = true) -> Host.MACOS + else -> Host.UNKNOWN } +} + +enum class Architecture(val serializedName: String) { + X86("x86"), + X64("x64"), + ARM("Arm"), + ARM64("Arm64"), +} - /** - * In order to support the Gradle Configuration Cache, this method must be called during - * the Configuration Phase in `afterEvaluate`. It isn't allowed to store a reference to - * the `project` property. - * - * This method is responsible for gathering all the analytics data we are sending. - */ - @Synchronized - fun collectAnalyticsData(project: Project) { - analytics = RealmAnalytics() - analytics!!.gatherAnalyticsDataIfNeeded(project) +val HOST_ARCH: String = run { + val hostArch = System.getProperty("os.arch") + when { + hostArch.contains("x86") && hostArch.contains("64") -> Architecture.X64.serializedName + hostArch.contains("x86") -> Architecture.X64.serializedName + hostArch.contains("aarch") && hostArch.contains("64") -> Architecture.ARM64.serializedName + hostArch.contains("aarch") -> Architecture.ARM.serializedName + else -> "Unknown[$hostArch]" } } diff --git a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/RealmAnalytics.kt b/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/RealmAnalytics.kt deleted file mode 100644 index 33e7f863d2..0000000000 --- a/packages/gradle-plugin/src/main/kotlin/io/realm/kotlin/gradle/analytics/RealmAnalytics.kt +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright 2021 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.realm.kotlin.gradle.analytics - -import com.android.build.gradle.BaseExtension -import io.realm.kotlin.gradle.RealmCompilerSubplugin -import org.gradle.api.Project -import org.gradle.api.logging.Logger -import org.gradle.api.logging.Logging -import java.io.File -import java.io.FileNotFoundException -import java.io.IOException -import java.io.UnsupportedEncodingException -import java.net.HttpURLConnection -import java.net.NetworkInterface -import java.net.SocketException -import java.net.URL -import java.security.MessageDigest -import java.security.NoSuchAlgorithmException -import java.util.Scanner -import javax.xml.bind.DatatypeConverter -import kotlin.experimental.and - -// Asynchronously submits build information to Realm when the gradle compile task run -// -// To be clear: this does *not* run when your app is in production or on -// your end-user's devices; it will only run when you build your app from source. -// -// Why are we doing this? Because it helps us build a better product for you. -// None of the data personally identifies you, your employer or your app, but it -// *will* help us understand what Realm version you use, what host OS you use, -// etc. Having this info will help with prioritizing our time, adding new -// features and deprecating old features. Collecting an anonymized bundle & -// anonymized MAC is the only way for us to count actual usage of the other -// metrics accurately. If we don't have a way to deduplicate the info reported, -// it will be useless, as a single developer building their app on Windows ten -// times would report 10 times more than a single developer that only builds -// once from Mac OS X, making the data all but useless. No one likes sharing -// data unless it's necessary, we get it, and we've debated adding this for a -// long long time. Since Realm is a free product without an email signup, we -// feel this is a necessary step so we can collect relevant data to build a -// better product for you. -// -// Currently the following information is reported: -// - What version of Realm is being used -// - What OS you are running on -// - An anonymized MAC address and bundle ID to aggregate the other information on. - -private const val TOKEN = "ce0fac19508f6c8f20066d345d360fd0" -private const val EVENT_NAME = "Run" -private const val URL_PREFIX = "https://data.mongodb-api.com/app/realmsdkmetrics-zmhtm/endpoint/metric_webhook/metric?data=" - -internal class RealmAnalytics { - - private val logger: Logger = Logging.getLogger("realm-analytics") - private var jsonData: String? = null - - companion object { - @Volatile - var METRIC_PROCESSED = false // prevent duplicate reports being sent from the same build run - } - - /** - * Collect analytics information. This method must be called in `project.afterEvaluate()`. - */ - public fun gatherAnalyticsDataIfNeeded(project: Project) { - val disableAnalytics: Boolean = project.gradle.startParameter.isOffline || "true".equals(System.getenv()["REALM_DISABLE_ANALYTICS"], ignoreCase = true) - if (!disableAnalytics) { - jsonData = jsonPayload(project) - // Resetting this flag as the Gradle Daemon keep this class and its state - // alive between builds, preventing analytics from being sent across multiple builds - METRIC_PROCESSED = false - } - } - - /** - * Send any previously gathered analytics data. [gatherAnalyticsDataIfNeeded] must be called - * first. - */ - @Synchronized - public fun sendAnalyticsData() { - if (!METRIC_PROCESSED) { - jsonData?.let { - logger.debug("Sending Realm analytics data: \n$jsonData") - sendAnalytics(it, logger) - } - METRIC_PROCESSED = true - } - } - - @Suppress("NestedBlockDepth") - private fun jsonPayload(project: Project): String { - val userId = ComputerIdentifierGenerator.get() - val appId = anonymousAppId(project) - val osType = System.getProperty("os.name") - val osVersion = System.getProperty("os.version") - - val projectAndroidExtension: BaseExtension? = - project.extensions.findByName("android") as BaseExtension? - val minSDK = projectAndroidExtension?.defaultConfig?.minSdkVersion?.apiString - val targetSDK = projectAndroidExtension?.defaultConfig?.targetSdkVersion?.apiString - - // We cannot use resolved configurations here as this code is called in - // afterEvaluate, and resolving it prevents other plugins from modifying - // them. E.g the KMP plugin will crash if we resolve the configurations - // in `afterEvaluate`. This means we can only see dependencies directly set, - // and not their transitive dependencies. This should be fine as we only - // want to track builds directly using Realm. - var usesSync = false - outer@ - for (conf in project.configurations) { - for (dependency in conf.dependencies) { - if (dependency.group == "io.realm.kotlin" && dependency.name == "library-sync") { - // In Java we can detect Sync through a Gradle configuration closure. - // In Kotlin, this choice is currently determined by which dependency - // people include - usesSync = true - break@outer - } - } - } - - // FIXME Improve metrics with details about targets, etc. - // https://github.com/realm/realm-kotlin/issues/127 - return """{ - "event": "$EVENT_NAME", - "properties": { - "token": "$TOKEN", - "distinct_id": "$userId", - "Anonymized MAC Address": "$userId", - "Anonymized Bundle ID": "$appId", - "Binding": "kotlin", - "Language": "kotlin", - "Realm Version": "${RealmCompilerSubplugin.version}", - "Sync Enabled": ${if (usesSync) "true" else "false"}, - "Host OS Type": "$osType", - "Host OS Version": "$osVersion", - "Target OS Minimum Version": "$minSDK", - "Target OS Version": "$targetSDK" - } - }""" - } - - private fun anonymousAppId(project: Project): String { - var projectName = project.rootProject.name - if (projectName.isEmpty()) { - projectName = project.name - } - - var packageName = project.group.toString() - if (packageName.isEmpty()) { - packageName = project.rootProject.group.toString() - } - - return hexStringify(sha256Hash("$packageName.$projectName".toByteArray())) - } - - @Suppress("TooGenericExceptionCaught") - private fun sendAnalytics(json: String, logger: Logger) { - try { - logger.debug("Sending analytics payload\n$json") - Thread( - Runnable { - try { - val response = networkQuery(json) - logger.debug("Analytics sent: $response") - } catch (e: InterruptedException) { - logger.debug("Sending analytics was interrupted.") - } - } - ).apply { - setDaemon(true) - }.start() - } catch (e: Exception) { - // Analytics failing for any reason should not crash the build - logger.debug("Error when sending: $e") - } - } - - private fun networkQuery(jsonPayload: String): Int { - try { - val url = URL(URL_PREFIX + base64Encode(jsonPayload)) - val connection = url.openConnection() as HttpURLConnection - connection.requestMethod = "GET" - connection.connect() - return connection.responseCode - } catch (ignored: Throwable) { - return -1 - } - } -} - -/** - * Generate a unique identifier for a computer. The method being used depends on the platform: - * - OS X: Mac address of en0 - * - Windows: BIOS identifier - * - Linux: Machine ID provided by the OS - */ -internal object ComputerIdentifierGenerator { - private const val UNKNOWN = "unknown" - private val OS = System.getProperty("os.name").lowercase() - @Suppress("TooGenericExceptionCaught", "SwallowedException") - fun get(): String { - return try { - when { - isWindows -> { - windowsIdentifier - } - isMac -> { - macOsIdentifier - } - isLinux -> { - linuxMacAddress - } - else -> { - UNKNOWN - } - } - } catch (e: Exception) { - UNKNOWN - } - } - - private val isWindows: Boolean - get() = OS.contains("win") - private val isMac: Boolean - get() = OS.contains("mac") - private val isLinux: Boolean - get() = OS.contains("inux") - - @get:Throws(FileNotFoundException::class, NoSuchAlgorithmException::class) - private val linuxMacAddress: String - get() { - var machineId = File("/var/lib/dbus/machine-id") - if (!machineId.exists()) { - machineId = File("/etc/machine-id") - } - if (!machineId.exists()) { - return UNKNOWN - } - var scanner: Scanner? = null - return try { - scanner = Scanner(machineId) - val id = scanner.useDelimiter("\\A").next() - hexStringify(sha256Hash(id.toByteArray())) - } finally { - scanner?.close() - } - } - - @get:Throws(SocketException::class, NoSuchAlgorithmException::class) - private val macOsIdentifier: String - get() { - val networkInterface = NetworkInterface.getByName("en0") - val hardwareAddress = networkInterface.hardwareAddress - return hexStringify(sha256Hash(hardwareAddress)) - } - - @get:Throws(IOException::class, NoSuchAlgorithmException::class) - private val windowsIdentifier: String - get() { - val runtime = Runtime.getRuntime() - val process = runtime.exec(arrayOf("wmic", "csproduct", "get", "UUID")) - var result: String? = null - val `is` = process.inputStream - val sc = Scanner(process.inputStream) - `is`.use { - while (sc.hasNext()) { - val next = sc.next() - if (next.contains("UUID")) { - result = sc.next().trim { it <= ' ' } - break - } - } - } - return if (result == null) UNKNOWN else hexStringify(sha256Hash(result!!.toByteArray())) - } -} - -/** - * Encode the given string with Base64 - * @param data the string to encode - * @return the encoded string - * @throws UnsupportedEncodingException - */ -@Throws(UnsupportedEncodingException::class) -internal fun base64Encode(data: String): String? { - return DatatypeConverter.printBase64Binary(data.toByteArray(charset("UTF-8"))) -} - -/** - * Compute the SHA-256 hash of the given byte array - * @param data the byte array to hash - * @return the hashed byte array - * @throws NoSuchAlgorithmException - */ -@Throws(NoSuchAlgorithmException::class) -internal fun sha256Hash(data: ByteArray?): ByteArray { - val messageDigest = MessageDigest.getInstance("SHA-256") - return messageDigest.digest(data) -} - -/** - * Convert a byte array to its hex-string - * @param data the byte array to convert - * @return the hex-string of the byte array - */ -@Suppress("MagicNumber") -internal fun hexStringify(data: ByteArray): String { - val stringBuilder = java.lang.StringBuilder() - for (singleByte: Byte in data) { - stringBuilder.append(((singleByte and 0xff.toByte()) + 0x100).toString(16).substring(1)) - } - return stringBuilder.toString() -} diff --git a/packages/plugin-compiler/src/main/kotlin/io/realm/kotlin/compiler/RealmCommandLineProcessor.kt b/packages/plugin-compiler/src/main/kotlin/io/realm/kotlin/compiler/RealmCommandLineProcessor.kt index cdb947458b..926eec3ed4 100644 --- a/packages/plugin-compiler/src/main/kotlin/io/realm/kotlin/compiler/RealmCommandLineProcessor.kt +++ b/packages/plugin-compiler/src/main/kotlin/io/realm/kotlin/compiler/RealmCommandLineProcessor.kt @@ -26,7 +26,9 @@ import org.jetbrains.kotlin.config.CompilerConfigurationKey // Must match io.realm.kotlin.gradle.RealmCompilerSubplugin.bundleId const val BUNDLE_ID_KEY = "bundleId" +const val FEATURE_LIST_PATH_KEY = "featureListPath" val bundleIdConfigurationKey: CompilerConfigurationKey = CompilerConfigurationKey("io.realm.kotlin.bundleId") +val featureListPathConfigurationKey: CompilerConfigurationKey = CompilerConfigurationKey("io.realm.kotlin.featureListPath") @OptIn(ExperimentalCompilerApi::class) @AutoService(CommandLineProcessor::class) @@ -39,7 +41,14 @@ class RealmCommandLineProcessor : CommandLineProcessor { valueDescription = "Anonymized Bundle Id", required = false, allowMultipleOccurrences = false - ) + ), + CliOption( + optionName = "featureListPath", + description = "Feature List Path", + valueDescription = "Feature List Path", + required = false, + allowMultipleOccurrences = false + ), ) override fun processOption( @@ -47,9 +56,11 @@ class RealmCommandLineProcessor : CommandLineProcessor { value: String, configuration: CompilerConfiguration ) { - when { - option.optionName == BUNDLE_ID_KEY -> + when (option.optionName) { + BUNDLE_ID_KEY -> configuration.put(bundleIdConfigurationKey, value) + FEATURE_LIST_PATH_KEY -> + configuration.put(featureListPathConfigurationKey, value) else -> super.processOption(option, value, configuration) } }