554829dc9e
Issue with previous code was that branch selected was applied to all repositories. To be able to control branch on a per-repo basis, we need to use codebases instead. This commit only enables branch (master/testbuild) and revision selection for master, we probably do not need this for submodules currently (easy to enable anyway). Note: tested on local buildbot setup, for linux only (up to build step, have no build env).
359 lines
12 KiB
Python
359 lines
12 KiB
Python
# -*- python -*-
|
|
# ex: set syntax=python:
|
|
|
|
# <pep8 compliant>
|
|
|
|
"""
|
|
Stock Twisted directory lister doesn't provide any information about last file
|
|
modification time, we hack the class a bit in order to have such functionaliity
|
|
:)
|
|
"""
|
|
|
|
from buildbot.status.web.base import DirectoryLister
|
|
|
|
|
|
def get_files_and_directories(self, directory):
|
|
from twisted.web.static import (getTypeAndEncoding,
|
|
formatFileSize)
|
|
import urllib
|
|
import cgi
|
|
import time
|
|
import os
|
|
files = []
|
|
dirs = []
|
|
for path in directory:
|
|
url = urllib.quote(path, "/")
|
|
escapedPath = cgi.escape(path)
|
|
lastmodified = time.ctime(os.path.getmtime(
|
|
os.path.join(self.path, path)))
|
|
if os.path.isdir(os.path.join(self.path, path)):
|
|
url = url + '/'
|
|
dirs.append({'text': escapedPath + "/", 'href': url,
|
|
'size': '', 'type': '[Directory]',
|
|
'encoding': '',
|
|
'lastmodified': lastmodified})
|
|
else:
|
|
mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
|
|
self.contentEncodings,
|
|
self.defaultType)
|
|
try:
|
|
size = os.stat(os.path.join(self.path, path)).st_size
|
|
except OSError:
|
|
continue
|
|
files.append({
|
|
'text': escapedPath, "href": url,
|
|
'type': '[%s]' % mimetype,
|
|
'encoding': (encoding and '[%s]' % encoding or ''),
|
|
'size': formatFileSize(size),
|
|
'lastmodified': lastmodified})
|
|
return dirs, files
|
|
DirectoryLister._getFilesAndDirectories = get_files_and_directories
|
|
|
|
# Dictionary that the buildmaster pays attention to.
|
|
c = BuildmasterConfig = {}
|
|
|
|
# BUILD SLAVES
|
|
#
|
|
# We load the slaves and their passwords from a separator file, so we can have
|
|
# this one in SVN.
|
|
|
|
from buildbot.buildslave import BuildSlave
|
|
import master_private
|
|
|
|
c['slaves'] = []
|
|
|
|
for slave in master_private.slaves:
|
|
c['slaves'].append(BuildSlave(slave['name'], slave['password']))
|
|
|
|
# TCP port through which slaves connect
|
|
|
|
c['slavePortnum'] = 9989
|
|
|
|
# CHANGE SOURCES
|
|
|
|
from buildbot.changes.svnpoller import SVNPoller
|
|
from buildbot.changes.gitpoller import GitPoller
|
|
|
|
c['change_source'] = GitPoller(
|
|
'git://git.blender.org/blender.git',
|
|
pollinterval=1200)
|
|
|
|
|
|
# CODEBASES
|
|
#
|
|
# Allow to controll separately things like branches for each repo and submodules.
|
|
|
|
all_repositories = {
|
|
r'git://git.blender.org/blender.git': 'blender',
|
|
r'git://git.blender.org/blender-translations.git': 'blender-translations',
|
|
r'git://git.blender.org/blender-addons.git': 'blender-addons',
|
|
r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
|
|
r'git://git.blender.org/scons.git': 'scons',
|
|
}
|
|
|
|
def codebaseGenerator(chdict):
|
|
return all_repositories[chdict['repository']]
|
|
|
|
c['codebaseGenerator'] = codebaseGenerator
|
|
|
|
|
|
# SCHEDULERS
|
|
#
|
|
# Decide how to react to incoming changes.
|
|
|
|
# from buildbot.scheduler import Scheduler
|
|
from buildbot.schedulers import timed, forcesched
|
|
|
|
c['schedulers'] = []
|
|
|
|
|
|
def schedule_force_build(name):
|
|
c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
|
|
builderNames=[name],
|
|
codebases=[forcesched.CodebaseParameter(
|
|
codebase="blender",
|
|
branch=forcesched.ChoiceStringParameter(
|
|
name="branch", choices=["master", "testbuild"], default="master"),
|
|
# Do not hide revision, can be handy!
|
|
repository=forcesched.FixedParameter(name="repository", default="", hide=True),
|
|
project=forcesched.FixedParameter(name="project", default="", hide=True)),
|
|
# For now, hide other codebases.
|
|
forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
|
|
forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
|
|
forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
|
|
forcesched.CodebaseParameter(hide=True, codebase="scons")],
|
|
properties=[]))
|
|
|
|
|
|
def schedule_build(name, hour, minute=0):
|
|
c['schedulers'].append(timed.Nightly(name='nightly ' + name,
|
|
codebases={
|
|
"blender": {"repository": ""},
|
|
"blender-translations": {"repository": ""},
|
|
"blender-addons": {"repository": ""},
|
|
"blender-addons-contrib": {"repository": ""},
|
|
"scons": {"repository": ""}},
|
|
branch=None,
|
|
builderNames=[name],
|
|
hour=hour,
|
|
minute=minute))
|
|
|
|
|
|
# BUILDERS
|
|
#
|
|
# The 'builders' list defines the Builders, which tell Buildbot how to
|
|
# perform a build: what steps, and which slaves can execute them.
|
|
# Note that any particular build will only take place on one slave.
|
|
|
|
from buildbot.process.factory import BuildFactory
|
|
from buildbot.steps.source import SVN
|
|
from buildbot.steps.source import Git
|
|
from buildbot.steps.shell import ShellCommand
|
|
from buildbot.steps.shell import Compile
|
|
from buildbot.steps.shell import Test
|
|
from buildbot.steps.transfer import FileUpload
|
|
from buildbot.steps.master import MasterShellCommand
|
|
from buildbot.config import BuilderConfig
|
|
|
|
# add builder utility
|
|
|
|
c['builders'] = []
|
|
buildernames = []
|
|
|
|
|
|
def add_builder(c, name, libdir, factory, branch='',
|
|
rsync=False, hour=3, minute=0):
|
|
slavenames = []
|
|
|
|
for slave in master_private.slaves:
|
|
if name in slave['builders']:
|
|
slavenames.append(slave['name'])
|
|
|
|
if len(slavenames) > 0:
|
|
f = factory(name, libdir, branch, rsync)
|
|
c['builders'].append(BuilderConfig(name=name,
|
|
slavenames=slavenames,
|
|
factory=f,
|
|
category='blender'))
|
|
buildernames.append(name)
|
|
|
|
schedule_build(name, hour, minute)
|
|
schedule_force_build(name)
|
|
|
|
# common steps
|
|
|
|
|
|
def git_submodule_step(submodule):
|
|
return Git(name=submodule + '.git',
|
|
repourl='git://git.blender.org/' + submodule + '.git',
|
|
mode='update',
|
|
codebase=submodule,
|
|
workdir=submodule + '.git')
|
|
|
|
|
|
def git_step(branch=''):
|
|
if branch:
|
|
return Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='update',
|
|
branch=branch,
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
else:
|
|
return Git(name='blender.git',
|
|
repourl='git://git.blender.org/blender.git',
|
|
mode='update',
|
|
codebase='blender',
|
|
workdir='blender.git',
|
|
submodules=True)
|
|
|
|
|
|
def git_submodules_update():
|
|
command = ['git', 'submodule', 'foreach', '--recursive',
|
|
'git', 'pull', 'origin', 'master']
|
|
return ShellCommand(name='Submodules Update',
|
|
command=command,
|
|
description='updating',
|
|
descriptionDone='up to date',
|
|
workdir='blender.git')
|
|
|
|
|
|
def lib_svn_step(dir):
|
|
return SVN(name='lib svn',
|
|
baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
|
|
mode='update',
|
|
defaultBranch='trunk',
|
|
workdir='lib/' + dir)
|
|
|
|
|
|
def rsync_step(id, branch, rsync_script):
|
|
return ShellCommand(name='rsync',
|
|
command=['python', rsync_script, id, branch],
|
|
description='uploading',
|
|
descriptionDone='uploaded',
|
|
workdir='install')
|
|
|
|
# generic builder
|
|
|
|
|
|
def generic_builder(id, libdir='', branch='', rsync=False):
|
|
filename = 'uploaded/buildbot_upload_' + id + '.zip'
|
|
compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
|
|
test_script = '../blender.git/build_files/buildbot/slave_test.py'
|
|
pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
|
|
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
|
|
unpack_script = 'master_unpack.py'
|
|
|
|
f = BuildFactory()
|
|
if libdir != '':
|
|
f.addStep(lib_svn_step(libdir))
|
|
|
|
for submodule in ('blender-translations',
|
|
'blender-addons',
|
|
'blender-addons-contrib',
|
|
'scons'):
|
|
f.addStep(git_submodule_step(submodule))
|
|
|
|
f.addStep(git_step(branch))
|
|
f.addStep(git_submodules_update())
|
|
|
|
f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
|
|
f.addStep(Test(command=['python', test_script, id]))
|
|
f.addStep(ShellCommand(name='package',
|
|
command=['python', pack_script, id, branch],
|
|
description='packaging',
|
|
descriptionDone='packaged'))
|
|
if rsync:
|
|
f.addStep(rsync_step(id, branch, rsync_script))
|
|
elif id.find('cmake') != -1:
|
|
f.addStep(FileUpload(name='upload',
|
|
slavesrc='buildbot_upload.zip',
|
|
masterdest=filename,
|
|
maxsize=150 * 1024 * 1024))
|
|
else:
|
|
f.addStep(FileUpload(name='upload',
|
|
slavesrc='buildbot_upload.zip',
|
|
masterdest=filename,
|
|
maxsize=150 * 1024 * 1024,
|
|
workdir='install'))
|
|
f.addStep(MasterShellCommand(name='unpack',
|
|
command=['python', unpack_script, filename],
|
|
description='unpacking',
|
|
descriptionDone='unpacked'))
|
|
return f
|
|
|
|
# builders
|
|
|
|
add_builder(c, 'mac_x86_64_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=5)
|
|
add_builder(c, 'mac_i386_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=11)
|
|
add_builder(c, 'linux_glibc211_i386_scons', '', generic_builder, hour=1)
|
|
add_builder(c, 'linux_glibc211_x86_64_scons', '', generic_builder, hour=2)
|
|
add_builder(c, 'win32_scons', 'windows', generic_builder, hour=1)
|
|
add_builder(c, 'win64_scons', 'win64', generic_builder, hour=2)
|
|
#add_builder(c, 'win32_scons_vc2012', 'windows_vc11', generic_builder, hour=1)
|
|
#add_builder(c, 'win64_scons_vc2012', 'win64_vc11', generic_builder, hour=2)
|
|
add_builder(c, 'win32_scons_vc2013', 'windows_vc12', generic_builder, hour=1)
|
|
add_builder(c, 'win64_scons_vc2013', 'win64_vc12', generic_builder, hour=2)
|
|
#add_builder(c, 'mingw_win32_scons', 'mingw32', generic_builder, hour=4)
|
|
add_builder(c, 'mingw_win64_scons', 'mingw64', generic_builder, hour=3)
|
|
#add_builder(c, 'freebsd_i386_cmake', '', generic_builder, hour=1)
|
|
#add_builder(c, 'freebsd_x86_64_cmake', '', generic_builder, hour=2)
|
|
|
|
# Multiview branch
|
|
add_builder(c, 'multiview_win64_scons', 'win64', generic_builder, 'multiview', hour=4)
|
|
add_builder(c, 'multiview_win32_scons', 'windows', generic_builder, 'multiview', hour=5)
|
|
|
|
# STATUS TARGETS
|
|
#
|
|
# 'status' is a list of Status Targets. The results of each build will be
|
|
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
|
|
# including web pages, email senders, and IRC bots.
|
|
|
|
c['status'] = []
|
|
|
|
from buildbot.status import html
|
|
from buildbot.status.web import authz
|
|
from buildbot.status.web import auth
|
|
|
|
users = []
|
|
for slave in master_private.slaves:
|
|
users += [(slave['name'], slave['password'])]
|
|
|
|
authz_cfg = authz.Authz(
|
|
auth=auth.BasicAuth(users),
|
|
# change any of these to True to enable; see the manual for more
|
|
# options
|
|
gracefulShutdown=False,
|
|
forceBuild=True, # use this to test your slave once it is set up
|
|
forceAllBuilds=False,
|
|
pingBuilder=False,
|
|
stopBuild=True,
|
|
stopAllBuilds=False,
|
|
cancelPendingBuild=True,
|
|
)
|
|
|
|
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
|
#c['status'].append(html.WebStatus(http_port=8010))
|
|
|
|
# PROJECT IDENTITY
|
|
|
|
c['projectName'] = "Blender"
|
|
c['projectURL'] = "http://www.blender.org"
|
|
|
|
# the 'buildbotURL' string should point to the location where the buildbot's
|
|
# internal web server (usually the html.WebStatus page) is visible. This
|
|
# typically uses the port number set in the Waterfall 'status' entry, but
|
|
# with an externally-visible host name which the buildbot cannot figure out
|
|
# without some help.
|
|
|
|
c['buildbotURL'] = "http://builder.blender.org/"
|
|
|
|
# DB URL
|
|
#
|
|
# This specifies what database buildbot uses to store change and scheduler
|
|
# state. You can leave this at its default for all but the largest
|
|
# installations.
|
|
|
|
c['db_url'] = "sqlite:///state.sqlite"
|