master.cfg

# -*- python -*-
# ex: set filetype=python:

import os

from buildbot.plugins import *
from buildbot.process.properties import Interpolate
from datetime import datetime

# This is the buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# Custom variables
REPO_URL="https://github.com/lquinet/meta-fullmetalupdate-extra.git"
SUPPORTED_MACHINES_ROCKO=["imx6qdlsabresd", "raspberrypi3"]
SUPPORTED_MACHINES_THUD=["imx8mqevk", "stm32mp1-disco"]
SUPPORTED_MACHINES_WARRIOR=["imx8mqevk"]
BRANCH_MACHINE_PAIRS = {
    "rocko" : SUPPORTED_MACHINES_ROCKO,
    "thud" : SUPPORTED_MACHINES_THUD,
    "warrior" : SUPPORTED_MACHINES_WARRIOR,
}
YOCTO_REPO_URL="https://github.com/lquinet/fullmetalupdate-yocto-demo.git"

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.

c['workers'] = [worker.Worker("example-worker", 'pass', max_builds=1)] # limit Concurrency to 1

if 'BUILDBOT_MQ_URL' in os.environ:
    c['mq'] = {
        'type' : 'wamp',
        'router_url': os.environ['BUILDBOT_MQ_URL'],
        'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
        'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
        'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
        'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
    }
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

gitpoller = changes.GitPoller(
        repourl=REPO_URL,
        workdir='gitpoller-workdir', 
        branches=True,
        pollInterval=60) # poll all branches from meta-fullmetalupdate-extra repo

c['change_source'] = [gitpoller]


####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.

c['builders'] = []

BUILDBOT_WORKER_WORKDIR=os.environ.get("BUILDBOT_WORKER_WORKDIR", "/tmp/buildbot")

# Create all builders depending of BRANCH_MACHINE_PAIRS
for branch, machines in BRANCH_MACHINE_PAIRS.items():
    for machine in machines:
        machine_branch = "{}_{}".format(machine, branch)

        # Create steps list
        all_steps = [
            # create directory for the branch/machine pair
            steps.ShellCommand(name="mkdir -p {}".format(machine_branch),
                            command="mkdir -p {}".format(machine_branch),
                            description="mkdir -p {}".format(machine_branch),
                            workdir="{}".format(BUILDBOT_WORKER_WORKDIR)), 
            # clone fullmetalupdate-yocto-demo
            steps.ShellCommand(name="clone fullmetalupdate-yocto-demo",
                            command="rm -rf fullmetalupdate-yocto-demo \
                            ; git clone {} \
                            && mkdir -p fullmetalupdate-yocto-demo/build".format(YOCTO_REPO_URL),
                            haltOnFailure=True,
                            description="clone fullmetalupdate-yocto-demo",
                            workdir="{}/{}".format(BUILDBOT_WORKER_WORKDIR, machine_branch)), 
            # Launch ConfigureBuild script
            steps.ShellCommand(name="ConfigureBuild",
                            command="./ConfigureBuild.sh",
                            haltOnFailure=True,
                            description="ConfigureBuild",
                            workdir="{}/{}/fullmetalupdate-yocto-demo".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),
            # Launch StartBuild.sh sync
            steps.ShellCommand(name="./StartBuild.sh sync {} {} dev".format(machine, branch),
                            command="./StartBuild.sh sync {} {} dev".format(machine, branch),
                            haltOnFailure=True,
                            description="./StartBuild.sh sync {} {} dev".format(machine, branch),
                            workdir="{}/{}/fullmetalupdate-yocto-demo".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),
            # Launch StartBuild.sh all
            steps.ShellCommand(name="./StartBuild.sh all",
                            command="./StartBuild.sh all",
                            haltOnFailure=True,
                            description="./StartBuild.sh all",
                            workdir="{}/{}/fullmetalupdate-yocto-demo".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),  
            # Copy image file
            steps.ShellCommand(name="Copy image",
                            command="mkdir -p images \
                            && cp fullmetalupdate-yocto-demo/build/yocto/build/tmp/fullmetalupdate-os/deploy/images/{}/fullmetalupdate-os-package-{}.wic \
                            images/fullmetalupdate-os-package-{}_{}.wic".format(machine, machine, machine, "$(date -u +\"%FT%H%M%SZ\")"),
                            haltOnFailure=True,
                            description="Copy image",
                            workdir="{}/{}".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),

            # Flash image
            steps.ShellCommand(name="Flash image",
                            command="dd if=$(ls -t | head -1) of=$SD_CARD_DEV_PATH bs=1M iflag=fullblock oflag=direct conv=fsync && sync",
                            haltOnFailure=True,
                            description="Flash image",
                            workdir="{}/{}/images".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),
        ]

        # build factory from steps
        factory = util.BuildFactory(all_steps)

        # append new builder
        c['builders'].append(util.BuilderConfig(name="{}_builder".format(machine_branch),
                                        workernames=["example-worker"],
                                        factory=factory))

# Create test builders
for branch, machines in BRANCH_MACHINE_PAIRS.items(): # first loop for branches
    for machine in machines: # second loop for machines
        machine_branch = "{}_{}".format(machine, branch)

        # Create steps list
        all_steps = [
            # Launch StartBuild.sh all
            steps.ShellCommand(name="./StartBuild.sh all",
                            command="./StartBuild.sh all",
                            haltOnFailure=True,
                            description="./StartBuild.sh all",
                            workdir="{}/{}/fullmetalupdate-yocto-demo".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),  
            # Copy image file
            steps.ShellCommand(name="Copy image",
                            command="mkdir -p images \
                            && cp fullmetalupdate-yocto-demo/build/yocto/build/tmp/fullmetalupdate-os/deploy/images/{}/fullmetalupdate-os-package-{}.wic \
                            images/fullmetalupdate-os-package-{}_$(date -u +\"%FT%H%M%SZ\").wic".format(machine, machine, machine, "$(date -u +\"%FT%H%M%SZ\")"),
                            haltOnFailure=True,
                            description="Copy image",
                            workdir="{}/{}".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),

            # Flash image
            steps.ShellCommand(name="Flash image",
                            command="dd if=$(ls -t | head -1) of=$SD_CARD_DEV_PATH bs=1M iflag=fullblock oflag=direct conv=fsync && sync",
                            haltOnFailure=True,
                            description="Flash image",
                            workdir="{}/{}/images".format(BUILDBOT_WORKER_WORKDIR, machine_branch)),           
        ]

        # build factory from steps
        factory = util.BuildFactory(all_steps)

        # append new builder
        c['builders'].append(util.BuilderConfig(name="{}_test_builder".format(machine_branch),
                                        workernames=["example-worker"],
                                        factory=factory))


####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes. 

c['schedulers'] = []

# Create one sheduler for each branch
for branch, machines in BRANCH_MACHINE_PAIRS.items():
    builderNames = []
    for machine in machines:
        builderNames.append("{}_{}_builder".format(machine, branch))
    c['schedulers'].append(
        schedulers.SingleBranchScheduler(
                            name="{}_meta-fullmetalupdate-extra_sched".format(branch),
                            change_filter=util.ChangeFilter(branch=branch),
                            treeStableTimer=60,
                            builderNames=builderNames)
    )

# Create a forced scheduler for each builder
for b in c['builders']:
    c['schedulers'].append(
        schedulers.ForceScheduler(
                                name="forced_sched_{}".format(b.name),
                                builderNames=[b.name])  
    ) 

# Create a test scheduler: run all tests builders if there is a change in test branch
builderNames = []
for branch, machines in BRANCH_MACHINE_PAIRS.items():  # first loop for branches
    for machine in machines:  # second loop for machines
        builderNames.append("{}_{}_test_builder".format(machine, branch))

c['schedulers'].append(
    schedulers.SingleBranchScheduler(
                                name="test_meta-fullmetalupdate-extra_sched",
                                change_filter=util.ChangeFilter(branch="test"),
                                treeStableTimer=60,
                                builderNames=builderNames)
)

####### BUILDBOT SERVICES (REPORTERS)

# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.

c['services'] = []

# publishes a build status using GitHub Status API
context = Interpolate("buildbot/%(prop:buildername)s")
gs = reporters.GitHubStatusPush(token=os.environ.get("GITHUB_TOKEN", ""),
                                context=context,
                                startDescription='Build started.',
                                endDescription='Build done.',
                                verbose=True)
c['services'].append(gs)

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').

c['title'] = "FMU automation build" 
c['titleURL'] = "https://github.com/lquinet/fmu-automation-build"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.

c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")

# minimalistic config to activate new web UI
# Buildbot badges plugin produces an image with information about the last build
# availabe at http://<buildbotURL>/badges/<buildername>.svg
c['www'] = dict(
    port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
    plugins=dict(
        waterfall_view={},
        console_view={},
        badges={})
    )

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
}