ci: create decision task to run os-integration tasks

requirements/local.txt

@@ -10,35 +10,31 @@ cachetools==5.5.0 \
     --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \
     --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a
     # via tox
-chardet==5.2.0 \
-    --hash=sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7 \
-    --hash=sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970
-    # via tox
 colorama==0.4.6 \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
     # via tox
-distlib==0.3.8 \
-    --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
-    --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
+distlib==0.3.9 \
+    --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
+    --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
     # via virtualenv
 filelock==3.16.1 \
     --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
     --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
     # via
     #   tox
     #   virtualenv
-pyproject-api==1.7.2 \
-    --hash=sha256:17c025105f8d27e22ffe542fe7dff3391b3736191a28294773a1f3b9ed25282b \
-    --hash=sha256:dc5b0e0f6e291a4f22b46e182c9c6d4915c62b1f089b8de1b73f2d06ae453593
+pyproject-api==1.8.0 \
+    --hash=sha256:3d7d347a047afe796fd5d1885b1e391ba29be7169bd2f102fcd378f04273d228 \
+    --hash=sha256:77b8049f2feb5d33eefcc21b57f1e279636277a8ac8ad6b5871037b243778496
     # via tox
-tox==4.19.0 \
-    --hash=sha256:66177d887f9d7ef8eaa9b58b187f7b865fa4c58650086c01336e82c9831e1867 \
-    --hash=sha256:6e20a520db7710f6980b8ec96bde189d6b8cf41b327ec703b03e1a2a447b1aaf
+tox==4.23.2 \
+    --hash=sha256:452bc32bb031f2282881a2118923176445bac783ab97c874b8770ab4c3b76c38 \
+    --hash=sha256:86075e00e555df6e82e74cfc333917f91ecb47ffbc868dcafbd2672e332f4a2c
     # via -r requirements/local.in
-virtualenv==20.26.5 \
-    --hash=sha256:4f3ac17b81fba3ce3bd6f4ead2749a72da5929c01774948e243db9ba41df4ff6 \
-    --hash=sha256:ce489cac131aa58f4b25e321d6d186171f78e6cb13fafbf32a840cee67733ff4
+virtualenv==20.27.1 \
+    --hash=sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba \
+    --hash=sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4
     # via tox
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/test.in

@@ -10,4 +10,5 @@ pip>=21.2
 pytest
 pytest-cov
 pytest-mock
+taskcluster-taskgraph
 yamllint

taskcluster/config.yml

@@ -6,6 +6,7 @@ treeherder:
 task-priority: low
 
 taskgraph:
+    register: fxci_config_taskgraph:register
     repositories:
         fxci:
             name: fxci-config

taskcluster/fxci_config_taskgraph/__init__.py

@@ -0,0 +1,17 @@
+from importlib import import_module
+
+
+def register(graph_config):
+    """Setup for task generation."""
+
+    # Import sibling modules, triggering decorators in the process
+    _import_modules(
+        [
+            "target_tasks",
+        ]
+    )
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(f".{module}", package=__name__)

taskcluster/fxci_config_taskgraph/target_tasks.py

@@ -0,0 +1,6 @@
+from taskgraph.target_tasks import register_target_task
+
+
+@register_target_task("os-integration")
+def os_integration(full_task_graph, parameters, graph_config):
+    return ["schedule-os-integration"]

taskcluster/kinds/schedule/kind.yml

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+  - tc-admin
+
+transforms:
+  - taskgraph.transforms.run:transforms
+  - taskgraph.transforms.task:transforms
+
+task-defaults:
+  run-on-tasks-for: []  # these tasks should only be triggered manually
+  run:
+    checkout: true
+    cwd: '{checkout}'
+    using: run-task
+  worker-type: t-linux
+  worker:
+    docker-image: {in-tree: python3.11}
+    max-run-time: 600
+
+tasks:
+  os-integration:
+    description: "Run Gecko os-integration tests"
+    dependencies:
+      apply: tc-admin-apply-staging
+    scopes:
+      - secrets:get:project/releng/fxci-config/taskcluster-stage-client
+    run:
+      command: >-
+        pip install --user -r requirements/test.txt &&
+        python taskcluster/scripts/schedule-tasks.py

taskcluster/kinds/tc-admin/kind.yml

@@ -23,7 +23,7 @@ tasks:
     apply-staging:
         description: "Run `tc-admin apply --envrionment=staging` to apply config to Taskcluster stage"
         scopes:
-          - secrets:get:project/releng/fxci-config/taskcluster-stage-client
+            - secrets:get:project/releng/fxci-config/taskcluster-stage-client
         run:
             command: >-
                 pip install --user -r requirements/base.txt &&

taskcluster/scripts/schedule-tasks.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from functools import cache
+from typing import Any
+
+import slugid
+import taskcluster
+from taskgraph.util.parameterization import resolve_timestamps
+from taskgraph.util.time import current_json_time
+
+DEFAULT_INDEX_PATHS = [
+    "gecko.v2.mozilla-central.latest.taskgraph.decision-os-integration"
+]
+FIREFOXCI_ROOT_URL = "https://firefox-ci-tc.services.mozilla.com"
+STAGING_ROOT_URL = "https://stage.taskcluster.nonprod.cloudops.mozgcp.net"
+
+
+@cache
+def get_taskcluster_client(service: str, staging=False):
+    if staging:
+        secrets = get_taskcluster_client("secrets")
+        result = secrets.get("project/releng/fxci-config/taskcluster-stage-client")
+        assert result
+
+        stage_auth = result["secret"]
+        assert isinstance(stage_auth, dict)
+        options = {"rootUrl": STAGING_ROOT_URL, "credentials": stage_auth}
+    elif "TASKCLUSTER_PROXY_URL" in os.environ:
+        options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
+    else:
+        options = taskcluster.optionsFromEnvironment()
+
+    return getattr(taskcluster, service.capitalize())(options)
+
+
+def find_tasks(index_paths: None | list[str] = None) -> list[dict[str, Any]]:
+    queue = get_taskcluster_client("queue")
+    index = get_taskcluster_client("index")
+
+    index_paths = index_paths or DEFAULT_INDEX_PATHS
+
+    tasks = []
+    for path in index_paths:
+        data = index.findTask(path)
+        assert data
+        task_id = data["taskId"]
+
+        task_graph = queue.getLatestArtifact(task_id, "public/task-graph.json")
+        assert task_graph
+
+        for task in task_graph.values():
+            assert isinstance(task, dict)
+            if task.get("attributes", {}).get("unittest_variant") != "os-integration":
+                continue
+
+            tasks.append(task["task"])
+
+    return tasks
+
+
+def rewrite_mounts(task_def: dict[str, Any]) -> None:
+    index = get_taskcluster_client("index")
+
+    for mount in task_def["payload"].get("mounts", []):
+        content = mount["content"]
+        if "artifact" not in content:
+            continue
+
+        if "namespace" in content:
+            task_id = index.findTask(content["namespace"])
+        else:
+            assert "taskId" in content
+            task_id = content["taskId"]
+
+        content["url"] = (
+            f"{FIREFOXCI_ROOT_URL}/api/queue/v1/task/{task_id}/artifacts/{content['artifact']}"
+        )
+
+        del content["artifact"]
+        if "namespace" in content:
+            del content["namespace"]
+        if "taskId" in content:
+            del content["taskId"]
+
+
+def schedule_task(task_def: dict[str, Any], task_id: str | None = None):
+    queue = get_taskcluster_client("queue")
+    queue_staging = get_taskcluster_client("queue", staging=True)
+    env = task_def["payload"].setdefault("env", {})
+
+    # We set firefoxci as the root url to trick the `fetch-content` script into
+    # downloading artifacts from the firefoxci instance. This only works for
+    # public artifacts.
+    if "MOZ_FETCHES" in env:
+        task_def["payload"]["command"].insert(
+            0, ["export", f"TASKCLUSTER_ROOT_URL={FIREFOXCI_ROOT_URL}"]
+        )
+
+    # Drop down to level 1 resources
+    for key in ("provisionerId", "taskQueueId", "schedulerId"):
+        if key in task_def:
+            task_def[key] = task_def[key].replace("3", "1")
+
+    task_def["taskGroupId"] = os.environ["TASK_ID"]
+    task_def["priority"] = "low"
+    rewrite_mounts(task_def)
+
+    del task_def["dependencies"]
+    del task_def["routes"]
+
+    # It's not possible to trick docker-worker into using image tasks from
+    # another Taskcluster instance, so we need to schedule those on the staging
+    # instance as well.
+    image = task_def["payload"].get("image")
+    if image and "taskId" in image:
+        try:
+            state = queue_staging.status(image["taskId"])["status"]["state"]
+            task_def["dependencies"] = [image["taskId"]]
+        except Exception:
+            state = "not found"
+
+        if state not in ("unscheduled", "pending", "running", "completed"):
+            image_task_def = queue.task(image["taskId"])
+            image_task_def["created"] = {"relative-datestamp": "0 seconds"}
+            image_task_def["deadline"] = {"relative-datestamp": "1 day"}
+            image_task_def["expires"] = {"relative-datestamp": "1 year"}
+
+            # We re-use the same taskId as the image task from the firefoxci
+            # instance. This gives us a hacky implementation of "cached-tasks".
+            schedule_task(image_task_def, task_id=image["taskId"])
+
+    now = current_json_time(datetime_format=True)
+    task_def = resolve_timestamps(now, task_def)
+
+    task_id = task_id or slugid.nice()
+    print(
+        f"Creating {task_def['metadata']['name']}: {STAGING_ROOT_URL}/tasks/{task_id}"
+    )
+    return queue_staging.createTask(task_id, task_def)
+
+
+def schedule_tasks():
+    for task_def in find_tasks():
+        schedule_task(task_def)
+
+
+if __name__ == "__main__":
+    sys.exit(schedule_tasks())