Stop relying on task routes specific to platform and configuration (mozilla#375)

Author: marco-c

bot/code_coverage_bot/artifacts.py

@@ -26,8 +26,8 @@
 
 
 class ArtifactsHandler(object):
-    def __init__(self, task_ids, parent_dir="ccov-artifacts", task_name_filter="*"):
-        self.task_ids = task_ids
+    def __init__(self, test_tasks, parent_dir="ccov-artifacts", task_name_filter="*"):
+        self.test_tasks = test_tasks
         self.parent_dir = parent_dir
         self.task_name_filter = task_name_filter
         self.artifacts = []
@@ -133,26 +133,9 @@ def is_filtered_task(self, task):
     def download_all(self):
         os.makedirs(self.parent_dir, exist_ok=True)
 
-        # The test tasks for the Linux and Windows builds are in the same group,
-        # but the following code is generic and supports build tasks split in
-        # separate groups.
-        groups = set(
-            [
-                taskcluster.get_task_details(build_task_id)["taskGroupId"]
-                for build_task_id in self.task_ids.values()
-                if build_task_id is not None
-            ]
-        )
-        test_tasks = [
-            task
-            for group in groups
-            for task in taskcluster.get_tasks_in_group(group)
-            if taskcluster.is_coverage_task(task["task"])
-            and not self.is_filtered_task(task)
-        ]
-        logger.info("Downloading artifacts from {} tasks".format(len(test_tasks)))
-
-        for test_task in test_tasks:
+        logger.info("Downloading artifacts from {} tasks".format(len(self.test_tasks)))
+
+        for test_task in self.test_tasks:
             status = test_task["status"]["state"]
             task_id = test_task["status"]["taskId"]
             while status not in FINISHED_STATUSES:
@@ -168,7 +151,7 @@ def download_all(self):
 
         # Choose best tasks to download (e.g. 'completed' is better than 'failed')
         download_tasks = {}
-        for test_task in test_tasks:
+        for test_task in self.test_tasks:
             status = test_task["status"]["state"]
             assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
                 status

bot/code_coverage_bot/hooks/base.py

@@ -20,9 +20,6 @@
 logger = structlog.get_logger(__name__)
 
 
-PLATFORMS = ["linux", "windows", "android-test", "android-emulator"]
-
-
 class Hook(object):
     def __init__(
         self,
@@ -54,21 +51,26 @@ def __init__(
         assert os.path.isdir(cache_root), f"Cache root {cache_root} is not a dir."
         self.repo_dir = os.path.join(cache_root, self.branch)
 
-        # Load current coverage task for all platforms
-        task_ids = {
-            platform: taskcluster.get_task(self.branch, self.revision, platform)
-            for platform in PLATFORMS
-        }
+        # Load coverage tasks for all platforms
+        decision_task_id = taskcluster.get_decision_task(self.branch, self.revision)
+
+        group = taskcluster.get_task_details(decision_task_id)["taskGroupId"]
+
+        test_tasks = [
+            task
+            for task in taskcluster.get_tasks_in_group(group)
+            if taskcluster.is_coverage_task(task["task"])
+        ]
 
         # Check the required platforms are present
+        platforms = set(
+            taskcluster.get_platform(test_task["task"]) for test_task in test_tasks
+        )
         for platform in required_platforms:
-            if not task_ids[platform]:
-                raise Exception(
-                    f"Code coverage build on {platform} failed and was not indexed."
-                )
+            assert platform in platforms, f"{platform} missing in the task group."
 
         self.artifactsHandler = ArtifactsHandler(
-            task_ids, self.artifacts_dir, task_name_filter
+            test_tasks, self.artifacts_dir, task_name_filter
         )
 
     @property

bot/code_coverage_bot/taskcluster.py

@@ -17,23 +17,8 @@
 NAME_PARTS_TO_SKIP = ("opt", "debug", "e10s", "1proc")
 
 
-def get_task(branch, revision, platform):
-    if platform == "linux":
-        platform_name = "linux64-ccov-opt"
-        product = "firefox"
-    elif platform == "windows":
-        platform_name = "win64-ccov-opt"
-        product = "firefox"
-    elif platform == "android-test":
-        platform_name = "android-test-ccov"
-        product = "mobile"
-    elif platform == "android-emulator":
-        platform_name = "android-api-16-ccov-debug"
-        product = "mobile"
-    else:
-        raise Exception(f"Unsupported platform: {platform}")
-
-    route = f"gecko.v2.{branch}.revision.{revision}.{product}.{platform_name}"
+def get_decision_task(branch, revision):
+    route = f"gecko.v2.{branch}.revision.{revision}.firefox.decision"
     index = taskcluster_config.get_service("index")
     try:
         return index.findTask(route)["taskId"]

bot/tests/conftest.py

@@ -68,8 +68,18 @@ def TASK_NOT_FOUND():
 
 
 @pytest.fixture(scope="session")
-def LATEST_LINUX():
-    return load_json("latest_linux.json")
+def DECISION_TASK_ID():
+    return "OuvSoOjkSvKYLbaGMknMfA"
+
+
+@pytest.fixture(scope="session")
+def DECISION_TASK():
+    return load_json("decision_task.json")
+
+
+@pytest.fixture(scope="session")
+def LATEST_DECISION():
+    return load_json("latest_decision.json")
 
 
 @pytest.fixture(scope="session")
@@ -92,16 +102,6 @@ def LINUX_TASK_ARTIFACTS():
     return load_json("linux_task_artifacts.json")
 
 
-@pytest.fixture(scope="session")
-def LATEST_WIN():
-    return load_json("latest_win.json")
-
-
-@pytest.fixture(scope="session")
-def WIN_TASK_ID():
-    return "PWnw3h-QQSiqxO83MDzKag"
-
-
 @pytest.fixture(scope="session")
 def GROUP_TASKS_1():
     return load_json("task-group_1.json")

bot/tests/fixtures/decision_task.json

@@ -0,0 +1,93 @@
+{
+  "provisionerId": "gecko-3",
+  "workerType": "decision",
+  "schedulerId": "gecko-level-3",
+  "taskGroupId": "OuvSoOjkSvKYLbaGMknMfA",
+  "dependencies": [],
+  "requires": "all-completed",
+  "routes": [
+    "tc-treeherder.v2.mozilla-central.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.37010",
+    "index.gecko.v2.mozilla-central.latest.taskgraph.decision",
+    "index.gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.taskgraph.decision",
+    "index.gecko.v2.mozilla-central.pushlog-id.37010.decision",
+    "[email protected]",
+    "[email protected]",
+    "index.gecko.v2.mozilla-central.latest.firefox.decision",
+    "index.gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision"
+  ],
+  "priority": "lowest",
+  "retries": 5,
+  "created": "2020-01-13T15:45:58.646Z",
+  "deadline": "2020-01-14T15:45:58.646Z",
+  "expires": "2021-01-12T15:45:58.646Z",
+  "scopes": [
+    "assume:repo:hg.mozilla.org/mozilla-central:branch:default",
+    "queue:route:[email protected].*",
+    "in-tree:hook-action:project-gecko/in-tree-action-3-*"
+  ],
+  "payload": {
+    "env": {
+      "GECKO_BASE_REPOSITORY": "https://hg.mozilla.org/mozilla-unified",
+      "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/mozilla-central",
+      "GECKO_HEAD_REF": "7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
+      "GECKO_HEAD_REV": "7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
+      "HG_STORE_PATH": "/builds/worker/checkouts/hg-store",
+      "TASKCLUSTER_CACHES": "/builds/worker/checkouts",
+      "PYTHONDONTWRITEBYTECODE": "1",
+      "TASKCLUSTER_ROOT_URL": "https://taskcluster.net",
+      "TASKCLUSTER_PROXY_URL": "http://taskcluster"
+    },
+    "cache": {
+      "gecko-level-3-checkouts-sparse-v2": "/builds/worker/checkouts"
+    },
+    "features": {
+      "taskclusterProxy": true,
+      "chainOfTrust": true
+    },
+    "image": "taskcluster/decision:2.2.0@sha256:0e9689e94605eb8395f5b49141a48148416b0d825f6f7be04c29642d1a85ee3d",
+    "maxRunTime": 1800,
+    "command": [
+      "/builds/worker/bin/run-task",
+      "--gecko-checkout=/builds/worker/checkouts/gecko",
+      "--gecko-sparse-profile=build/sparse-profiles/taskgraph",
+      "--",
+      "bash",
+      "-cx",
+      "cd /builds/worker/checkouts/gecko && ln -s /builds/worker/artifacts artifacts && ./mach --log-no-times taskgraph decision --pushlog-id='37010' --pushdate='1578930324' --project='mozilla-central' --owner='[email protected]' --level='3' --tasks-for='hg-push' --base-repository=\"$GECKO_BASE_REPOSITORY\" --head-repository=\"$GECKO_HEAD_REPOSITORY\" --head-ref=\"$GECKO_HEAD_REF\" --head-rev=\"$GECKO_HEAD_REV\" \n"
+    ],
+    "artifacts": {
+      "public": {
+        "type": "directory",
+        "path": "/builds/worker/artifacts",
+        "expires": "2021-01-12T15:45:58.646Z"
+      }
+    }
+  },
+  "metadata": {
+    "owner": "[email protected]",
+    "source": "https://hg.mozilla.org/mozilla-central/raw-file/7828a10a94b6afb78d18d9b7b83e7aa79337cc24/.taskcluster.yml",
+    "name": "Gecko Decision Task",
+    "description": "The task that creates all of the other tasks in the task graph"
+  },
+  "tags": {
+    "createdForUser": "[email protected]",
+    "kind": "decision-task"
+  },
+  "extra": {
+    "treeherder": {
+      "machine": {
+        "platform": "gecko-decision"
+      },
+      "symbol": "D"
+    },
+    "tasks_for": "hg-push",
+    "notify": {
+      "email": {
+        "link": {
+          "text": "Treeherder Jobs",
+          "href": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&revision=7828a10a94b6afb78d18d9b7b83e7aa79337cc24"
+        }
+      }
+    }
+  }
+}

bot/tests/fixtures/latest_decision.json

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision",
+  "taskId": "OuvSoOjkSvKYLbaGMknMfA",
+  "rank": 0,
+  "data": {},
+  "expires": "2021-01-12T15:45:58.646Z"
+}

bot/tests/fixtures/latest_linux.json

@@ -9,6 +9,7 @@
 
 from code_coverage_bot.artifacts import Artifact
 from code_coverage_bot.artifacts import ArtifactsHandler
+from code_coverage_bot.hooks.base import Hook
 
 FILES = [
     "windows_mochitest-1_code-coverage-jsvm.info",
@@ -259,28 +260,45 @@ def build_task(task_state):
 
 
 def test_download_all(
-    LINUX_TASK_ID,
-    LINUX_TASK,
+    DECISION_TASK_ID,
+    DECISION_TASK,
+    LATEST_DECISION,
     GROUP_TASKS_1,
     GROUP_TASKS_2,
     fake_artifacts,
     mock_taskcluster,
+    tmpdir,
 ):
     responses.add(
         responses.GET,
-        f"http://taskcluster.test/api/queue/v1/task/{LINUX_TASK_ID}",
-        json=LINUX_TASK,
+        "http://taskcluster.test/api/index/v1/task/gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision",
+        json=LATEST_DECISION,
+        status=200,
+    )
+    responses.add(
+        responses.GET,
+        f"http://taskcluster.test/api/queue/v1/task/{DECISION_TASK_ID}",
+        json=DECISION_TASK,
         status=200,
     )
     for group_tasks in _group_tasks():
         responses.add(
             responses.GET,
-            "http://taskcluster.test/api/queue/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list",
+            "http://taskcluster.test/api/queue/v1/task-group/OuvSoOjkSvKYLbaGMknMfA/list",
             json=group_tasks,
             status=200,
         )
 
-        a = ArtifactsHandler({"linux": LINUX_TASK_ID})
+        h = Hook(
+            "https://hg.mozilla.org/mozilla-central",
+            "7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
+            "*",
+            tmpdir,
+            tmpdir,
+        )
+        a = h.artifactsHandler
+
+        # a = ArtifactsHandler({"linux": LINUX_TASK_ID})
 
         downloaded = set()