Buildbot initial configuration files, for http://builder.blender.org

Maintained in svn to make it easier for others to contribute changes,
actually updating builder.blender.org requires manual update on the
server.
This commit is contained in:
Brecht Van Lommel 2011-03-21 16:46:26 +00:00
parent f0d7fbc4b7
commit 9a75ba4463
3 changed files with 407 additions and 0 deletions

@ -0,0 +1,203 @@
# -*- python -*-
# ex: set syntax=python:
# Dictionary that the buildmaster pays attention to.
c = BuildmasterConfig = {}
# BUILD SLAVES
#
# We load the slaves and their passwords from a separator file, so we can have
# this one in SVN.
from buildbot.buildslave import BuildSlave
import master_private
c['slaves'] = []
for slave in master_private.slaves:
c['slaves'].append(BuildSlave(slave['name'], slave['password']))
# TCP port through which slaves connect
c['slavePortnum'] = 9989
# CHANGE SOURCES
from buildbot.changes.svnpoller import SVNPoller
c['change_source'] = SVNPoller(
'https://svn.blender.org/svnroot/bf-blender/trunk/',
pollinterval=1200)
# BUILDERS
#
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
from buildbot.process.factory import BuildFactory
from buildbot.steps.source import SVN
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import Compile
from buildbot.steps.shell import Test
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import FileDownload
from buildbot.steps.master import MasterShellCommand
from buildbot.config import BuilderConfig
# add builder utility
c['builders'] = []
buildernames = []
def add_builder(c, name, factory):
slavenames = []
for slave in master_private.slaves:
if name in slave['builders']:
slavenames.append(slave['name'])
f = factory(name)
c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
buildernames.append(name)
# common steps
def svn_step():
return SVN(baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/blender', mode='update', defaultBranch='trunk', workdir='blender')
def lib_svn_step(dir):
return SVN(name='lib svn', baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, mode='update', defaultBranch='trunk', workdir='lib/' + dir)
def cmake_compile_step():
return Compile(command=['make'], workdir='blender')
def cmake_test_step():
return Test(workdir='blender') # make test
def scons_compile_step():
return Compile(command=['python', 'scons/scons.py'], workdir='blender')
class SlavePack(ShellCommand):
pack_script = 'slave_pack.py'
def start(self):
if self.getProperty('buildername').find('scons')>=0:
self.setCommand(['python', pack_script, 'scons'])
else:
self.setCommand(['python', pack_script, 'cmake'])
ShellCommand.start(self)
def file_upload(f, id):
filename = 'buildbot_upload_' + id + '.zip'
pack_script = 'slave_pack.py'
unpack_script = 'master_unpack.py'
f.addStep(FileDownload(name='download', mastersrc=pack_script, slavedest=pack_script))
f.addStep(ShellCommand(name='package', command=['python', pack_script], description='packaging', descriptionDone='packaged'))
f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024))
f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
# linux cmake
def linux_cmake(id):
f = BuildFactory()
f.addStep(svn_step())
f.addStep(cmake_compile_step())
f.addStep(cmake_test_step())
file_upload(f, id)
return f
add_builder(c, 'linux_x86_64_cmake', linux_cmake)
# mac cmake
def mac_cmake(id):
f = BuildFactory()
f.addStep(svn_step())
f.addStep(lib_svn_step('darwin-9.x.universal'))
f.addStep(cmake_compile_step())
f.addStep(cmake_test_step())
file_upload(f, id)
return f
add_builder(c, 'mac_x86_64_cmake', mac_cmake)
# win32 scons
# TODO: add scons test target
def win32_scons(id):
f = BuildFactory()
f.addStep(svn_step())
f.addStep(lib_svn_step('windows'))
f.addStep(scons_compile_step())
file_upload(f, id)
return f
add_builder(c, 'win32_scons', win32_scons)
# SCHEDULERS
#
# Decide how to react to incoming changes.
from buildbot.scheduler import Scheduler
from buildbot.schedulers import timed
c['schedulers'] = []
#c['schedulers'].append(Scheduler(name="all", branch=None,
# treeStableTimer=None,
# builderNames=[]))
#c['schedulers'].append(timed.Periodic(name="nightly",
# builderNames=buildernames,
# periodicBuildTimer=24*60*60))
c['schedulers'].append(timed.Nightly(name='nightly',
builderNames=buildernames,
hour=3,
minute=0))
# STATUS TARGETS
#
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
c['status'] = []
from buildbot.status import html
from buildbot.status.web import auth, authz
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
gracefulShutdown = False,
forceBuild = True, # use this to test your slave once it is set up
forceAllBuilds = False,
pingBuilder = False,
stopBuild = False,
stopAllBuilds = False,
cancelPendingBuild = False,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
# PROJECT IDENTITY
c['projectName'] = "Blender"
c['projectURL'] = "http://www.blender.org"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
c['buildbotURL'] = "http://builder.blender.org/"
# DB URL
#
# This specifies what database buildbot uses to store change and scheduler
# state. You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"

@ -0,0 +1,117 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# Runs on Buildbot master, to unpack incoming unload.zip into latest
# builds directory and remove older builds.
import os
import shutil
import sys
import zipfile
# extension stripping
def strip_extension(filename):
extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
for ext in extensions:
if filename.endswith(ext):
filename = filename[:-len(ext)]
return filename
# extract platform from package name
def get_platform(filename):
# name is blender-version-platform.extension. we want to get the
# platform out, but there may be some variations, so we fiddle a
# bit to handle current and hopefully future names
filename = strip_extension(filename)
filename = strip_extension(filename)
tokens = filename.split("-")
platforms = ['osx', 'mac', 'bsd', 'windows', 'linux', 'source', 'irix', 'solaris']
platform_tokens = []
found = False
for i, token in enumerate(tokens):
if not found:
for platform in platforms:
if token.lower().find(platform) != -1:
found = True
if found:
platform_tokens += [token]
return '-'.join(platform_tokens)
# get filename
if len(sys.argv) < 2:
sys.stderr.write("Not enough arguments, expecting file to unpack\n")
sys.exit(1)
filename = sys.argv[1]
# open zip file
if not os.path.exists(filename):
sys.stderr.write("File " + filename + " not found.\n")
sys.exit(1)
try:
z = zipfile.ZipFile(filename, "r")
except Exception, ex:
sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
sys.exit(1)
if len(z.namelist()) != 1:
sys.stderr.write("Expected on file in " + filename + ".")
sys.exit(1)
package = z.namelist()[0]
packagename = os.path.basename(package)
# detect platform
platform = get_platform(packagename)
if platform == '':
sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
sys.exit(1)
# extract
dir = 'public_html/latest_builds'
try:
zf = z.open(package)
f = file(os.path.join(dir, packagename), "wb")
shutil.copyfileobj(zf, f)
zf.close()
z.close()
except Exception, ex:
sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
sys.exit(1)
# remove other files from the same platform
try:
for f in os.listdir(dir):
if f.lower().find(platform.lower()) != -1:
if f != packagename:
os.remove(os.path.join(dir, f))
except Exception, ex:
sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
sys.exit(1)

@ -0,0 +1,87 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# Runs on buildbot slave, creating a release package using the build
# system and zipping it into buildbot_upload.zip. This is then uploaded
# to the master in the next buildbot step.
import os
import subprocess
import sys
import zipfile
# find release directory
def find_release_directory():
for d in os.listdir('.'):
if os.path.isdir(d):
rd = os.path.join(d, 'release')
if os.path.exists(rd):
return rd
return None
# clean release directory if it already exists
dir = find_release_directory()
if dir:
for f in os.listdir(dir):
if os.path.isfile(os.path.join(dir, f)):
os.remove(os.path.join(dir, f))
# create release package
try:
os.chdir('../blender')
subprocess.call(['make', 'package_archive'])
os.chdir('../build')
except Exception, ex:
sys.stderr.write('Make package release failed' + str(ex) + '\n')
sys.exit(1)
# find release directory, must exist this time
dir = find_release_directory()
if not dir:
sys.stderr.write("Failed to find release directory.\n")
sys.exit(1)
# find release package
file = None
filepath = None
for f in os.listdir(dir):
rf = os.path.join(dir, f)
if os.path.isfile(rf) and f.startswith('blender'):
file = f
filepath = rf
if not file:
sys.stderr.write("Failed to find release package.\n")
sys.exit(1)
# create zip file
try:
upload_zip = "buildbot_upload.zip"
if os.path.exists(upload_zip):
os.remove(upload_zip)
z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
z.write(filepath, arcname=file)
z.close()
except Exception, ex:
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
sys.exit(1)