master.cfg

# -*- python -*-
# ex: set filetype=python:

from buildbot.plugins import *
import os
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.


def load_password():
    """ read username:password from password file """
    lines = open('password', 'r').readlines()
    usernames_passwords = []
    admin_usernames = []
    for line in lines:
        username, pwhash = line.split(':')
        entry = [username, pwhash.rstrip()]
        usernames_passwords.append(entry)
        admin_usernames.append(username)
    return usernames_passwords, admin_usernames

usernames_passwords, admin_usernames = load_password()

def load_mail_password():
    """ read worker information from workers file """
    with open('mail_password', 'r') as f:
        mail_password = f.readline()
    return mail_password

mail_password = load_mail_password()

def load_worker():
    """ read worker information from workers file """
    lines = open('workers.cfg', 'r').readlines()
    workers_info = []
    for line in lines:
        username, pwhash = line.split(':')
        entry = [username, pwhash.rstrip()]
        workers_info.append(entry)
    return workers_info

workers_info = load_worker()

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.

c['workers'] = []
for name, password in workers_info:
    c['workers'].append(worker.Worker(name, password))

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(changes.GitHubPullrequestPoller(
    owner='gentoo',
    branches=None,
    repo='linux-patches'))

c['change_source'].append(changes.GitHubPullrequestPoller(
    owner='gentoo',
    branches=None,
    category='gentoo-pull',
    repo='gentoo'))

c['change_source'].append(changes.GitPoller(
    repourl='https://github.com/gentoo/gentoo.git',
    branches=True,
    category="gentoo-pull",
    pollinterval=10))

c['change_source'].append(changes.GitPoller(
    repourl='git://anongit.gentoo.org/proj/linux-patches.git',
    only_tags=True,
    category="gentoo-tags-git"))

c['change_source'].append(changes.GitPoller(
    repourl='git://anongit.gentoo.org/proj/linux-patches.git',
    branches=True,
    category="gentoo-git"))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.
# In this case, just kick off a 'runtests' build


def change_files_json_push(change):
    print("Change111: "+str(change.files))
    if any("sys-kernel/gentoo-sources" in s for s in change.files):
        print("sys-kernel ebuild to test")
        return True
    else:
        return False

def syskernel_change(change):
    print("Change111: "+str(change.files))
    if any("sys-kernel/" in s for s in change.files):
        print("sys-kernel ebuild to test")
        return True
    else:
        return False

def eclass_change(change):
    print("Change111: "+str(change.files))
    if any("eclass/kernel-2.eclass" in s for s in change.files):
        print("sys-kernel ebuild to test")
        return True
    else:
        return False

c['schedulers'] = []

architecture_testing_list = ['amd64', 'arm']
branches_list = ['4.20', '4.19', '4.18', '4.14',
                 '4.9', '4.8', '4.4', '4.1']

def builderNames(branch):
    builders = set()

    for arch in architecture_testing_list:
        builders.add(branch + ':' + arch)

    return list(builders)

for branch in branches_list:
    c['schedulers'].append(schedulers.SingleBranchScheduler(
                                name=branch,
                                change_filter=util.ChangeFilter(branch=branch),
                                treeStableTimer=None,
                                builderNames=builderNames(branch)))
    # add a changefilter for the pull requests
    cf = util.ChangeFilter(category='pull', branch=branch)
    # but only those that are targeted for that branch
    cf.checks["prop:github.base.ref"] = cf.checks['branch']
    del cf.checks['branch']
    c['schedulers'].append(schedulers.SingleBranchScheduler(
                                name="pull" + branch,
                                change_filter=cf,
                                treeStableTimer=None,
                                builderNames=builderNames(branch)))

    stab_cf = util.ChangeFilter(category='pull', branch=branch)
    c['schedulers'].append(schedulers.SingleBranchScheduler(
                                name="stabilize" + branch,
                                change_filter=stab_cf,
                                treeStableTimer=None,
                                builderNames=builderNames(branch)))

    # add a changefilter for the pull requests
    gcf = util.ChangeFilter(category='gentoo-git', branch_re=branch+"\..*")
    c['schedulers'].append(schedulers.SingleBranchScheduler(
        name="git_pull" + branch,
        change_filter=gcf,
        treeStableTimer=None,
        builderNames=builderNames(branch)))

    stab_gcf = util.ChangeFilter(category='gentoo-tags-git',
                                 branch_re="refs/tags/" + branch + "_stabilize")
    c['schedulers'].append(schedulers.SingleBranchScheduler(
        name="git_stabilize" + branch,
        change_filter=stab_gcf,
        treeStableTimer=None,
        builderNames=builderNames(branch)))

gpcf = util.ChangeFilter(category='gentoo-pull', filter_fn=change_files_json_push)
c['schedulers'].append(schedulers.SingleBranchScheduler(
        name="gentoo_sources",
        change_filter=gpcf,
        treeStableTimer=None,
        builderNames=["gentoo_sources"]))
c['schedulers'].append(schedulers.ForceScheduler(
        name="force_gentoo_sources",
        builderNames=["gentoo_sources"]))

gpcf = util.ChangeFilter(category='gentoo-pull', filter_fn=syskernel_change)
c['schedulers'].append(schedulers.SingleBranchScheduler(
        name="other_sources",
        change_filter=gpcf,
        treeStableTimer=None,
        builderNames=["other_sources"]))
c['schedulers'].append(schedulers.ForceScheduler(
        name="force_other_sources",
        builderNames=["other_sources"]))

gpcf = util.ChangeFilter(category='gentoo-pull', filter_fn=eclass_change)
c['schedulers'].append(schedulers.SingleBranchScheduler(
        name="eclass_change",
        change_filter=gpcf,
        treeStableTimer=None,
        builderNames=["eclass_change"]))
c['schedulers'].append(schedulers.ForceScheduler(
        name="force_eclass_change",
        builderNames=["eclass_change"]))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.
from buildbot.plugins import reporters, util
from buildbot.process.properties import Interpolate

context = Interpolate("buildbot/%(prop:buildername)s")
gs = reporters.GitHubStatusPush(token=os.environ.get("BUILDBOT_STATUS_TOKEN"),
                                context=context,
                                startDescription='Build started.',
                                endDescription='Build done.',
                                verbose=True)


def download_new_patch_and_build_kernel(version, arch):
    factory = util.BuildFactory()
    factory.addStep(steps.GitHub(name="Fetching linux-patches repository",
                                 repourl='https://github.com/gentoo/linux-patches',
                                 mode='incremental',
                                 workdir="build/linux-patches", branch=version))

    factory.addStep(steps.FileDownload(mastersrc="~/files/check-kernelpage.py",
                                       workerdest="files/check-kernelpage.py"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/qemu_check.py",
                                       workerdest="files/qemu_check.py"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/patch-kernel.sh",
                                       workerdest="files/patch-kernel.sh"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/build-kernel.sh",
                                       workerdest="files/build-kernel.sh"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/clean.sh",
                                       workerdest="clean.sh"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/clean.py",
                                       workerdest="clean.py"))

    factory.addStep(steps.FileDownload(mastersrc="~/files/find.sh",
                                       workerdest="files/find.sh"))

    factory.addStep(steps.ShellCommand(name="Looking for new upstream release",
                                       command=["/usr/bin/python",
                                                "check-kernelpage.py",
                                                "--version", version],
                                       workdir="build/files/"))

    factory.addStep(steps.ShellCommand(name="Patching kernel",
                                       command=["/bin/sh", "patch-kernel.sh",
                                                "-a", arch, "-k", version],
                                       workdir="build/files/",
                                       haltOnFailure=True))

    factory.addStep(steps.ShellCommand(name="Listing rejected files",
                                       command=["/bin/sh", "find.sh"],
                                       workdir="build/files/"))

    factory.addStep(steps.ShellCommand(name="Building kernel",
                                       command=["/bin/sh", "../build-kernel.sh", arch],
                                       workdir="build/files/linux-" + version + "/",
                                       haltOnFailure=True))

    factory.addStep(steps.ShellCommand(name="Building modules",
                                       command=["/bin/sh", "../build-kernel.sh", arch, "modules"],
                                       workdir="build/files/linux-" + version + "/",
                                       haltOnFailure=True))

    factory.addStep(steps.ShellCommand(name="Booting with QEMU",
                                       command=["/usr/bin/python", "qemu_check.py", arch,
                                                util.Property('workername'), util.Property('buildnumber')],
                                       workdir="build/files/", timeout=3600))

    factory.addStep(steps.ShellCommand(name="Cleanup",
                                       command=["/bin/sh", "clean.sh"],
                                       workdir="build/", alwaysRun=True))

    return factory


@util.renderer
def filterFiles(props):
    files = props.getBuild().allFiles()
    print(files)
    build_files = [s for s in files if "sys-kernel/" in s]
    command = ["/usr/bin/python", "stabilize-packages.py"]
    for file in build_files:
        command.append(file)
    print(str(command))
    return command

@util.renderer
def pull_repourl(props):
    pull_repourl = props.getProperty('repository')
    return pull_repourl

def test_gentoo_sources():
    factory = util.BuildFactory()
    factory.addStep(steps.GitHub(name="Fetching repository",
                                 repourl=pull_repourl,
                                 mode='incremental', workdir="build/gentoo", shallow=50))
    factory.addStep(steps.FileDownload(mastersrc="~/files/qemu_check.py",
                                       workerdest="files/qemu_check.py"))
    factory.addStep(steps.FileDownload(mastersrc="~/files/stabilize-packages.py",
                                       workerdest="files/stabilize-packages.py"))
    factory.addStep(steps.FileDownload(mastersrc="~/files/clean.sh",
                                       workerdest="clean.sh"))
    factory.addStep(steps.FileDownload(mastersrc="~/files/clean.py",
                                       workerdest="clean.py"))
    factory.addStep(steps.ShellCommand(name="Stabilizing package",
                                       command=filterFiles,
                                       workdir="build/files/"))
    # factory.addStep(steps.ShellCommand(command=["/usr/bin/python", "qemu_check.py"],
    #                                workdir="build/files/"))
    factory.addStep(steps.ShellCommand(name="Cleanup",
                                       command=["/bin/sh", "clean.sh"],
                                       workdir="build/", alwaysRun=True))
    return factory


c['builders'] = []
for kernel_branch in branches_list:
    for arch in architecture_testing_list:
        c['builders'].append(
              util.BuilderConfig(name=kernel_branch + ':' + arch,
                                 workernames=["kernelci"],
                        factory=download_new_patch_and_build_kernel(kernel_branch, arch)))

c['builders'].append(
    util.BuilderConfig(name='gentoo_sources',
                       workernames=["kernelci"],
                       factory=test_gentoo_sources()))

c['builders'].append(
    util.BuilderConfig(name='other_sources',
                       workernames=["kernelci"],
                       factory=test_gentoo_sources()))

c['builders'].append(
    util.BuilderConfig(name='eclass_change',
                       workernames=["kernelci"],
                       factory=test_gentoo_sources()))

####### BUILDBOT SERVICES

# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.

c['services'] = []

irc = reporters.IRC(host="irc.freenode.net", nick="kernelci",
                    channels=["#gentoo-kernel"],
                    notify_events={
                        'started': 1,
                        'success': 1,
                        'failure': 1,
                        'exception': 1,
                    })
c['services'].append(irc)

template = u'''\
<h4>Build status: {{ summary }}</h4>
<p> Worker used: {{ workername }}</p>
{% for step in build['steps'] %}
<p> {{ step['name'] }}: {{ step['result'] }}</p>
{% endfor %}
<p><b> -- Gentoo Kernel-CI Buildbot</b></p>
'''

mn = reporters.MailNotifier(fromaddr="kernelCI-buildbot@gentoo.org",
                            sendToInterestedUsers=False,
                            mode=('all'),
                            extraRecipients=['alice.ferrazzi@gmail.com',
                                             'm.j.everitt@iee.org'],
                            messageFormatter=reporters.MessageFormatter(
                               template=template, template_type='html',
                               wantProperties=True, wantSteps=True),
                               useTls=True, relayhost="smtp.gmail.com",
                               smtpPort=587, smtpUser="gkernelci@gmail.com",
                               smtpPassword=mail_password)

c['services'].append(mn)
c['services'].append(gs)

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').

c['title'] = "Gentoo Kernel"
c['titleURL'] = "https://wiki.gentoo.org/wiki/Project:Kernel"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.

c['buildbotURL'] = "http://kernel1.amd64.dev.gentoo.org:8010/"

# minimalistic config to activate new web UI
c['www'] = dict(port=8010,
                plugins=dict(waterfall_view={}, console_view={}))
c['www']['authz'] = util.Authz(
    allowRules=[
        util.AnyEndpointMatcher(role="admins", defaultDeny=False),
        util.AnyControlEndpointMatcher(role="admins")
    ],
    roleMatchers=[
        util.RolesFromUsername(roles=['admins'], usernames=admin_usernames)
    ]
)

PR_props = [
    "github.base.ref",
    "github.number",
    "github.mergeable",

]

c['www']['allowed_origins'] = ["*"]

c['www']['change_hook_dialects'] = {
    "github": {
        "secret": os.environ.get("BUILDBOT_HOOK_SECRET"),
        "strict": True,
        "github_property_whitelist": PR_props
    }
}



c['www']['auth'] = util.UserPasswordAuth(usernames_passwords)

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can
    # leave this at its default for all but the largest installations.
    'db_url': "sqlite:///state.sqlite",
}