Use pkl to generate gha workflows (#3603)

* Replace pr.yml with one generated from pkl

* Generate wrappers.yml using pkl

* Add a space

* Fix platform name for uwp

* Move deploy baas to pr.yml

* Move cleanup-baas

* Actually commit the changes

* Move test-net-framework

* Use download-artifact@v4

* Use upload artifact v4

* Move net core tests

* Move test-weaver

* add --no-self-contained

* fix paths

* Move SG tests

* Move tvOS

* migrate iOS and macOS test jobs

* Move android, fix catalyst

* Actually use the android steps

* Migrate code coverage

* try to fix some build failures

* Disable some tests on AOT targets

* Revert some changes

* Fix some tests

* Try to use the original action

* Update setup-java action

* Ignore a few tests temporarily

* Migrate unity builds

* fix unity yaml

* Try to use newer java

* Add correct needs for nuget package

* Use JDK 17

* Migrate codeql, publish-prerelease

* Fix some workflows

* Migrate publish-release.yml

* Migrate main.pkl

* Change .NET verison of DeployApps to 8.x

* Revert core change

* Remove some redundant types

.github/pkl-workflows/GithubAction/PklProject

@@ -0,0 +1,8 @@
+amends "pkl:Project"
+
+package {
+  name = "realm-pkl-gha"
+  version = "1.0.0"
+  baseUri = "package://realm.io/realm-pkl-gha"
+  packageZipUrl = "https://realm.io/realm-pkl-gha@\(version).zip"
+}

.github/pkl-workflows/GithubAction/PklProject.deps.json

@@ -0,0 +1,4 @@
+{
+  "schemaVersion": 1,
+  "resolvedDependencies": {}
+}

.github/pkl-workflows/PklProject

@@ -0,0 +1,5 @@
+amends "pkl:Project"
+
+dependencies {
+  ["realm-gha"] = import("GithubAction/PklProject")
+}

.github/pkl-workflows/PklProject.deps.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": 1,
+  "resolvedDependencies": {
+    "package://realm.io/realm-pkl-gha@1": {
+      "type": "local",
+      "uri": "projectpackage://realm.io/[email protected]",
+      "path": "GithubAction"
+    }
+  }
+}

.github/pkl-workflows/Readme.md

@@ -0,0 +1,20 @@
+# Preprocessing the GA workflows
+
+We're using [pkl](http://github.com/apple/pkl) to generate the github actions workflows.
+
+## Prerequisites
+
+Install pkl: https://pkl-lang.org/main/current/pkl-cli/index.html#installation
+
+For macOS, this is simply
+
+```bash
+brew install pkl
+```
+
+## Building the workflows
+
+```bash
+cd $SolutionDir/.github/pkl-workflows
+for file in *.pkl ; do pkl eval $file -o  ../workflows/$(echo $file | sed s/pkl/yml/) ; done
+```

.github/pkl-workflows/codeql.pkl

@@ -0,0 +1,77 @@
+module codeql
+
+amends "GithubAction/GithubAction.pkl"
+
+import "helpers/Common.pkl"
+import "helpers/Steps.pkl"
+import "helpers/Actions.pkl"
+
+name = "CodeQL"
+on {
+  push {
+    branches {
+      Common.mainBranch
+    }
+  }
+  pull_request {
+    branches {
+      Common.mainBranch
+    }
+    paths {
+      "**.cs"
+      "**.cpp"
+      "**.hpp"
+      "**.csproj"
+      ".github/workflows/codeql.yml"
+    }
+  }
+}
+env {
+  ["REALM_DISABLE_ANALYTICS"] = true
+}
+concurrency {
+  group = "codeql-${{ github.head_ref || github.run_id }}"
+  `cancel-in-progress` = true
+}
+jobs {
+  ["analyze-csharp"] = new Job {
+    name = "Analyze C#"
+    `runs-on` = new WindowsLatest{}
+    permissions {
+      actions = "read"
+      contents = "read"
+      `security-events` = "write"
+    }
+    steps {
+      Steps.checkoutWithoutMatchers(false)
+      new {
+        name = "Initialize CodeQL"
+        uses = Actions.codeQLInit
+        with {
+          ["languages"] = "csharp"
+          ["config"] = """
+            queries:
+              - uses: security-and-quality
+            query-filters:
+              - exclude:
+                  id: cs/call-to-unmanaged-code
+              - exclude:
+                  id: cs/unmanaged-code
+            """
+        }
+      }
+      ...Steps.setupAndroid()
+      ...Steps.setupWorkloads("tvos ios maccatalyst android", "8.0.x")
+      ...Steps.msbuild(new Steps.MSBuildConfig {
+        projects = Common.packages.map((pkg) -> "Realm/\(pkg)").toListing()
+        properties {
+          ["UseSharedCompilation"] = "false"
+        }
+      })
+      new {
+        name = "Perform CodeQL Analysis"
+        uses = Actions.codeQLAnalyze
+      }
+    }
+  }
+}

.github/pkl-workflows/helpers/Actions.pkl

@@ -0,0 +1,24 @@
+module actions
+
+const runSimulator = "realm/ci-actions/run-ios-simulator@6418e15ed9bbdb19b7d456a347e5623779f95cdf"
+const setupJDK = "actions/setup-java@2e74cbce18569d23ca8b812590dbb83f13ac7c5a"
+const awsConfigureCredentials = "aws-actions/[email protected]"
+const coveralls = "coverallsapp/[email protected]"
+const checkout = "actions/checkout@v4"
+const uploadArtifact = "actions/upload-artifact@v4"
+const downloadArtifact = "actions/download-artifact@v4"
+const downloadAllArtifacts = "dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe" // 3.1.4
+const setupXcode = "maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd"
+const setupDotnet = "actions/setup-dotnet@5d1464d5da459f3d7085106d52e499f4dc5d0f59"
+const setupMsBuild = "microsoft/setup-msbuild@70b70342ae97ca98d5eaad06cafd26d30f9592a9"
+const setupAndroid = "android-actions/setup-android@e1f5280adf78cf863c0fa43ffabc64a9cd08153f"
+const setupNode = "actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8" // 4.0.2
+const restoreCache = "actions/cache/restore@v4"
+const cache = "actions/cache@v4"
+const codeQLInit = "github/codeql-action/init@a57c67b89589d2d13d5ac85a9fc4679c7539f94c"
+const codeQLAnalyze = "github/codeql-action/analyze@a57c67b89589d2d13d5ac85a9fc4679c7539f94c"
+const findPR = "juliangruber/find-pull-request-action@48b6133aa6c826f267ebd33aa2d29470f9d9e7d0" // 1.9.0
+const mergePR = "juliangruber/merge-pull-request-action@9234b8714dda9a08f3d1df5b2a6a3abd7b695353" // 1.3.1
+const createPR = "peter-evans/create-pull-request@6d6857d36972b65feb161a90e484f2984215f83e" // 6.0.5
+const publishGithubRelease = "ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5" // 1.14.0
+const releaseToSlack = "realm/ci-actions/release-to-slack@6418e15ed9bbdb19b7d456a347e5623779f95cdf"

.github/pkl-workflows/helpers/BaaS.pkl

@@ -0,0 +1,47 @@
+module baas
+
+import "../GithubAction/GithubAction.pkl" as gha
+import "Common.pkl"
+import "Steps.pkl"
+
+function deploy(differentiators: Listing<Common.SyncDifferentiator>): gha.MatrixJob = new {
+  name = "Deploy BaaS"
+  `runs-on` = new gha.UbuntuLatest{}
+  strategy {
+    matrix {
+      ["differentiator"] = differentiators
+    }
+  }
+  `if` = Common.ifNotCanceledCondition
+  steps {
+    ...Steps.checkout(false)
+    Steps.setupDotnet("8.0.x")
+    ...deployStep("${{ matrix.differentiator }}", true)
+  }
+}
+
+function deployStep(differentiator: Common.SyncDifferentiator?, shouldDeploy: Boolean): List<gha.Step> = if (shouldDeploy && differentiator != null) List(new gha.Step {
+  name = "Deploy Apps"
+  run = "dotnet run deploy-apps --baasaas-api-key=${{ secrets.BAASAAS_API_KEY }} --baas-differentiator=\(differentiator)-${{ github.run_id }}-${{ github.run_attempt }}"
+  `working-directory` = "Tools/DeployApps"
+}) else List()
+
+function cleanup(differentiators: Listing<Common.SyncDifferentiator>): gha.MatrixJob = new {
+  name = "Cleanup BaaS"
+  `runs-on` = new gha.UbuntuLatest{}
+  strategy {
+    matrix {
+      ["differentiator"] = differentiators
+    }
+  }
+  `if` = Common.ifNotCanceledCondition
+  steps {
+    ...Steps.checkout(false)
+    Steps.setupDotnet("8.0.x")
+    new {
+      name = "Terminate Baas"
+      run = "dotnet run terminate-baas --baasaas-api-key=${{ secrets.BAASAAS_API_KEY }} --baas-differentiator=${{ matrix.differentiator }}-${{ github.run_id }}-${{ github.run_attempt }}"
+      `working-directory` = "Tools/DeployApps"
+    }
+  }
+}

.github/pkl-workflows/helpers/Common.pkl

@@ -0,0 +1,90 @@
+module common
+
+import "../GithubAction/GithubAction.pkl" as gha
+import "BaaS.pkl"
+import "Lint.pkl"
+import "Package.pkl"
+import "Test.pkl" as TestJobs
+
+const configuration: String = "Release"
+
+const linuxArchs: List<String> = List("x86_64", "armhf", "aarch64")
+const applePlatforms: List<String> = List("iOS", "tvOS")
+const androidABIs: List<String> = List("armeabi-v7a", "arm64-v8a", "x86", "x86_64")
+const windowsArchs: List<String> = List("Win32", "x64", "ARM64")
+const uwpArchs: List<String> = List("Win32", "x64", "ARM", "ARM64")
+
+const appleTargets: List<String> = List("Device", "Simulator")
+
+const job_Packages: String = "build-packages"
+const job_Baas: String = "deploy-baas"
+const job_Unity: String = "build-unity"
+
+local const job_Wrappers: String = "build-wrappers"
+
+const mainBranch: String = "main"
+
+typealias NetFramework = "net6.0" | "net8.0" | String(startsWith("${{ matrix")) | String(startsWith("net8"))
+typealias NetRuntime = "win-x64" | "linux-x64" | "osx-x64" | "osx-arm64" | String(startsWith("${{ matrix"))
+typealias SyncDifferentiator = "ios-maui" | "android-maui" | "tvos" | "macos-maui" | "code-coverage" | "uwp" | "net-framework" | String(startsWith("${{ matrix"))
+
+const wrapperBinaryNames: List<String> =
+  List("macos", "catalyst")
+  + linuxArchs.map((arch) -> "linux-\(arch)")
+  + androidABIs.map((abi) -> "android-\(abi)")
+  + windowsArchs.map((arch) -> "windows-\(arch)")
+  + uwpArchs.map((arch) -> "windows-uwp-\(arch)")
+  + applePlatformTargets((platform, target) -> "\(platform)-\(target)")
+
+const defaultEnv: Mapping<String, String | Boolean> = new {
+  ["REALM_DISABLE_ANALYTICS"] = true
+  ["DOTNET_NOLOGO"] = true
+}
+
+const function applePlatformTargets(_transform: (String, String) -> String): List<String> = applePlatforms.map((platform) -> appleTargets.map((target) -> _transform.apply(platform, target))).flatten()
+
+const ifNotCanceledCondition = "always() && !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')"
+
+const nugetPackages: List<String> = List("Realm", "Realm.PlatformHelpers")
+const packages: List<String> = nugetPackages + List("Realm.UnityUtils", "Realm.UnityWeaver")
+
+const testTimeout: Int = 60
+
+const function defaultBuildJobs(baasDifferentiators: Listing<SyncDifferentiator>, netCoreVersions: Listing<NetFramework>): Mapping<String, gha.JobBase> = new {
+  [job_Wrappers] = new gha.ReusableWorkflowJob {
+    uses = "./.github/workflows/wrappers.yml"
+    name = "Wrappers"
+  }
+  [job_Baas] = BaaS.deploy(baasDifferentiators)
+  [job_Packages] = (Package.nuget("contains(github.head_ref, 'release')")){
+    needs {
+      job_Wrappers
+    }
+  }
+  [job_Unity] = Package.unity()
+  ...TestJobs.unity(new TestJobs.UnityTestConfig {
+    os = "linux"
+  })
+  ...TestJobs.unity(new TestJobs.UnityTestConfig {
+    os = "windows"
+  })
+  ["test-net-framework"] = TestJobs.netFramework(baasDifferentiators)
+  ["test-uwp"] = TestJobs.uwp(baasDifferentiators)
+  ["test-net-core"] = TestJobs.netCore(netCoreVersions)
+  ["test-macos-xamarin"] = TestJobs.macOS_Xamarin()
+  ["test-macos-maui"] = TestJobs.macOS_Maui(baasDifferentiators)
+  ["test-ios-xamarin"] = TestJobs.iOS_Xamarin()
+  ["test-ios-maui"] = TestJobs.iOS_Maui(baasDifferentiators)
+  ["test-tvos"] = TestJobs.tvOS(baasDifferentiators)
+  ["test-android-xamarin"] = TestJobs.android_Xamarin()
+  ["test-android-maui"] = TestJobs.android_Maui(baasDifferentiators)
+  ["test-woven-classes"] = TestJobs.wovenClasses()
+  ["test-source-generation"] = TestJobs.sourceGeneration()
+  ["test-weaver"] = TestJobs.weaver()
+  ["test-code-coverage"] = TestJobs.codeCoverage(job_Wrappers, baasDifferentiators)
+  ["cleanup-baas"] = (BaaS.cleanup(baasDifferentiators)) {
+    needs = baasDifferentiators.toList().map((d) -> "test-\(d)").toListing()
+  }
+  ["verify-namespaces"] = Lint.verifyNamespaces()
+  ["lint"] = Lint.lint()
+}

.github/pkl-workflows/helpers/Lint.pkl

@@ -0,0 +1,69 @@
+module lint
+
+import "../GithubAction/GithubAction.pkl" as gha
+import "Steps.pkl"
+import "Common.pkl"
+
+const actionVerifyTODO = "nirinchev/verify-todo@ffa352ac028b4cdc8bc626471d33aa341b3ab7c9"
+
+function lint(): gha.Job = new {
+  name = "Verify TODOs"
+  `runs-on` = new gha.UbuntuLatest{}
+  steps {
+    Steps.checkoutWithoutMatchers(false)
+    new gha.Step {
+      uses = actionVerifyTODO
+      with {
+        ["token"] = "${{ secrets.GITHUB_TOKEN }}"
+        ["include"] = "**/*.+(cs|cpp|hpp)"
+        ["exclude"] = "wrappers/realm-core/**"
+        ["pattern"] = "\\\\WR[A-Z]+-[0-9]+"
+      }
+    }
+  }
+}
+
+function verifyNamespaces(): gha.Job = new {
+  `runs-on` = new gha.UbuntuLatest{}
+  name = "Verify Namespaces"
+  needs {
+    Common.job_Packages
+  }
+  steps {
+    Steps.checkoutWithoutMatchers(false)
+    ...Steps.fetchPackages(null)
+    new {
+      run = "dotnet tool install ilspycmd -g --version 8.0.0.7345"
+    }
+    new {
+      name = "Verify Namespaces"
+      run = """
+        $isFailure = $false
+        Get-ChildItem ./Realm/packages -Filter *.nupkg | Foreach-Object {
+          $targetPath = Join-Path ./Realm/packages $_.BaseName
+          Expand-Archive $_.FullName -DestinationPath $targetPath
+
+          Get-ChildItem $targetPath -Filter *.dll -Recurse | ForEach-Object {
+            if (-not ($_.FullName -match "runtimes")) {
+              $ilspyOutput = ilspycmd $_.FullName
+
+              $parentDll = $_.FullName
+
+              $ilspyOutput | ForEach-Object {
+                if ($_ -match "namespace.*Realm(\\.|$)") {
+                  Write-Output "::error file=$parentDll::Realm present in namespace - $($_)"
+                  Set-Variable "isFailure" -Value $true
+                }
+              }
+            }
+          }
+        }
+
+        if ($isFailure) {
+          exit 1
+        }
+        """
+      shell = "pwsh"
+    }
+  }
+}