How to use taskgraph - 10 common examples

#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

from taskgraph.util.treeherder import split_symbol, join_symbol, add_suffix
from taskgraph.transforms.base import TransformSequence

transforms = TransformSequence()


def _remove_suffix(text, suffix):
    """
    Removes a suffix from a string.
    """
    if text.endswith(suffix):
        _drop = len(suffix) * -1
        text = text[:_drop]
    return text


@transforms.add
def tests_drop_1proc(config, jobs):
    """
    Remove the -1proc suffix from Treeherder group symbols.

Restore the -e10s suffix (because some day we will have them!)

    Reverses the effects of bug 1541527. Thunderbird builds are all single
    process.
    """
    for job in jobs:
        test = job['run']['test']
        e10s = test['e10s']

        if not e10s:  # test-name & friends end with '-1proc'
            test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
            test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = _remove_suffix(group, '-1proc')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] = job['label'].replace('-1proc', '')
            job['name'] = _remove_suffix(job['name'], '-1proc')
            job['treeherder']['symbol'] = test['treeherder-symbol']
        else:  # e10s in the future
            test['test-name'] = add_suffix(test['test-name'], '-e10s')
            test['try-name'] = add_suffix(test['try-name'], '-e10s')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = add_suffix(group, '-e10s')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] += '-e10s'
            job['name'] = add_suffix(job['name'], '-e10s')
            job['treeherder']['symbol'] = test['treeherder-symbol']

test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = _remove_suffix(group, '-1proc')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] = job['label'].replace('-1proc', '')
            job['name'] = _remove_suffix(job['name'], '-1proc')
            job['treeherder']['symbol'] = test['treeherder-symbol']
        else:  # e10s in the future
            test['test-name'] = add_suffix(test['test-name'], '-e10s')
            test['try-name'] = add_suffix(test['try-name'], '-e10s')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = add_suffix(group, '-e10s')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] += '-e10s'
            job['name'] = add_suffix(job['name'], '-e10s')
            job['treeherder']['symbol'] = test['treeherder-symbol']

        yield job

e10s = test['e10s']

        if not e10s:  # test-name & friends end with '-1proc'
            test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
            test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = _remove_suffix(group, '-1proc')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] = job['label'].replace('-1proc', '')
            job['name'] = _remove_suffix(job['name'], '-1proc')
            job['treeherder']['symbol'] = test['treeherder-symbol']
        else:  # e10s in the future
            test['test-name'] = add_suffix(test['test-name'], '-e10s')
            test['try-name'] = add_suffix(test['try-name'], '-e10s')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = add_suffix(group, '-e10s')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] += '-e10s'
            job['name'] = add_suffix(job['name'], '-e10s')
            job['treeherder']['symbol'] = test['treeherder-symbol']

        yield job

test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
            test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = _remove_suffix(group, '-1proc')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] = job['label'].replace('-1proc', '')
            job['name'] = _remove_suffix(job['name'], '-1proc')
            job['treeherder']['symbol'] = test['treeherder-symbol']
        else:  # e10s in the future
            test['test-name'] = add_suffix(test['test-name'], '-e10s')
            test['try-name'] = add_suffix(test['try-name'], '-e10s')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = add_suffix(group, '-e10s')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] += '-e10s'
            job['name'] = add_suffix(job['name'], '-e10s')
            job['treeherder']['symbol'] = test['treeherder-symbol']

        yield job

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""

from __future__ import absolute_import, print_function, unicode_literals

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by


transforms = TransformSequence()


@transforms.add
def resolve_keys(config, tasks):
    for task in tasks:
        for key in ("index", "worker-type", "worker.signing-type", "signing-format"):
            resolve_keyed_by(
                task,
                key,
                item_name=task["name"],
                **{
                    'build-type': task["attributes"]["build-type"],
                    'level': config.params["level"],
                }
            )
        yield task

# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Handle notifications like emails.
"""

from __future__ import absolute_import, print_function, unicode_literals

import copy
import json

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.util.schema import resolve_keyed_by

transforms = TransformSequence()


@transforms.add
def add_notify_email(config, tasks):
    for task in tasks:
        notify = task.pop('notify', {})
        email_config = notify.get('email')
        if email_config:
            extra = task.setdefault('extra', {})
            notify = extra.setdefault('notify', {})
            notify['email'] = {
                'content': email_config['content'],
                'subject': email_config['subject'],
                'link': email_config.get('link', None),
            }

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import os

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by

transforms = TransformSequence()


NOTIFY_EMAIL_ADDRESS = 'firefox-tv@mozilla.com'


@transforms.add
def email_task(config, tasks):
    tag = config.params.get("head_tag")
    release_type = config.params.get("release_type")

    for task in tasks:
        resolve_keyed_by(task, "worker.content", task["name"], **{
            "release-type": release_type or "production"
        })

        if release_type == 'lat':

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

from taskgraph.transforms.base import TransformSequence


transforms = TransformSequence()


@transforms.add
def signing_task(config, tasks):
    for task in tasks:
        task["worker"]["signing-type"] = 'dep-signing' if config.params["level"] != u'3' else 'production-signing'
        yield task


@transforms.add
def set_signing_format(config, tasks):
    for task in tasks:
        for upstream_artifact in task["worker"]["upstream-artifacts"]:
            upstream_artifact["formats"] = ["autograph_apk"]
        yield task

# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""

from __future__ import absolute_import, print_function, unicode_literals

import datetime

from taskgraph.transforms.base import TransformSequence
from fenix_taskgraph.gradle import get_variant
from fenix_taskgraph.util import upper_case_first_letter


transforms = TransformSequence()


@transforms.add
def add_variant_config(config, tasks):
    for task in tasks:
        attributes = task.setdefault("attributes", {})
        if not attributes.get("build-type"):
            attributes["build-type"] = task["name"]

        yield task


@transforms.add
def add_shippable_secrets(config, tasks):
    for task in tasks:
        secrets = task["run"].setdefault("secrets", [])