forked from bartvdbraak/blender
Buildbot: Remove master configuration files
Those are stored in blender-buildbot repository now, so having them in Blender only causes extra work to keep files in sync.
This commit is contained in:
Sergey Sharybin
parent
07f05e294a
commit
e24dee435e
@ -1,368 +0,0 @@
|
||||
# -*- python -*-
|
||||
# ex: set syntax=python:
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# List of the branches being built automatically overnight
|
||||
NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
|
||||
|
||||
# List of the branches available for force build
|
||||
FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
|
||||
|
||||
"""
|
||||
Stock Twisted directory lister doesn't provide any information about last file
|
||||
modification time, we hack the class a bit in order to have such functionaliity
|
||||
:)
|
||||
"""
|
||||
|
||||
from buildbot.status.web.base import DirectoryLister
|
||||
|
||||
|
||||
def get_files_and_directories(self, directory):
|
||||
from twisted.web.static import (getTypeAndEncoding,
|
||||
formatFileSize)
|
||||
import urllib
|
||||
import cgi
|
||||
import time
|
||||
import os
|
||||
files = []
|
||||
dirs = []
|
||||
for path in directory:
|
||||
url = urllib.quote(path, "/")
|
||||
escapedPath = cgi.escape(path)
|
||||
lastmodified = time.ctime(os.path.getmtime(
|
||||
os.path.join(self.path, path)))
|
||||
if os.path.isdir(os.path.join(self.path, path)):
|
||||
url = url + '/'
|
||||
dirs.append({'text': escapedPath + "/", 'href': url,
|
||||
'size': '', 'type': '[Directory]',
|
||||
'encoding': '',
|
||||
'lastmodified': lastmodified})
|
||||
else:
|
||||
mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
|
||||
self.contentEncodings,
|
||||
self.defaultType)
|
||||
try:
|
||||
size = os.stat(os.path.join(self.path, path)).st_size
|
||||
except OSError:
|
||||
continue
|
||||
files.append({
|
||||
'text': escapedPath, "href": url,
|
||||
'type': '[%s]' % mimetype,
|
||||
'encoding': (encoding and '[%s]' % encoding or ''),
|
||||
'size': formatFileSize(size),
|
||||
'lastmodified': lastmodified})
|
||||
return dirs, files
|
||||
DirectoryLister._getFilesAndDirectories = get_files_and_directories
|
||||
|
||||
# Dictionary that the buildmaster pays attention to.
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
# BUILD SLAVES
|
||||
#
|
||||
# We load the slaves and their passwords from a separator file, so we can have
|
||||
# this one in SVN.
|
||||
|
||||
from buildbot.buildslave import BuildSlave
|
||||
import master_private
|
||||
|
||||
c['slaves'] = []
|
||||
|
||||
for slave in master_private.slaves:
|
||||
c['slaves'].append(BuildSlave(slave['name'], slave['password']))
|
||||
|
||||
# TCP port through which slaves connect
|
||||
|
||||
c['slavePortnum'] = 9989
|
||||
|
||||
# CHANGE SOURCES
|
||||
|
||||
from buildbot.changes.svnpoller import SVNPoller
|
||||
from buildbot.changes.gitpoller import GitPoller
|
||||
|
||||
c['change_source'] = GitPoller(
|
||||
'git://git.blender.org/blender.git',
|
||||
pollinterval=1200)
|
||||
|
||||
|
||||
# CODEBASES
|
||||
#
|
||||
# Allow to control separately things like branches for each repo and submodules.
|
||||
|
||||
all_repositories = {
|
||||
r'git://git.blender.org/blender.git': 'blender',
|
||||
r'git://git.blender.org/blender-translations.git': 'blender-translations',
|
||||
r'git://git.blender.org/blender-addons.git': 'blender-addons',
|
||||
r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
|
||||
r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
|
||||
r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
|
||||
}
|
||||
|
||||
|
||||
def codebaseGenerator(chdict):
|
||||
return all_repositories[chdict['repository']]
|
||||
|
||||
c['codebaseGenerator'] = codebaseGenerator
|
||||
|
||||
|
||||
# SCHEDULERS
|
||||
#
|
||||
# Decide how to react to incoming changes.
|
||||
|
||||
# from buildbot.scheduler import Scheduler
|
||||
from buildbot.schedulers import timed, forcesched
|
||||
|
||||
c['schedulers'] = []
|
||||
|
||||
|
||||
def schedule_force_build(name):
|
||||
c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
|
||||
builderNames=[name],
|
||||
codebases=[forcesched.CodebaseParameter(
|
||||
codebase="blender",
|
||||
branch=forcesched.ChoiceStringParameter(
|
||||
name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
|
||||
# Do not hide revision, can be handy!
|
||||
repository=forcesched.FixedParameter(name="repository", default="", hide=True),
|
||||
project=forcesched.FixedParameter(name="project", default="", hide=True)),
|
||||
# For now, hide other codebases.
|
||||
forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
|
||||
forcesched.CodebaseParameter(
|
||||
codebase="blender-addons",
|
||||
branch=forcesched.ChoiceStringParameter(
|
||||
name="branch", choices=["master", "blender2.8"], default="master"),
|
||||
repository=forcesched.FixedParameter(name="repository", default="", hide=True),
|
||||
project=forcesched.FixedParameter(name="project", default="", hide=True),
|
||||
revision=forcesched.FixedParameter(name="revision", default="", hide=True),
|
||||
),
|
||||
forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
|
||||
forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
|
||||
forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
|
||||
properties=[]))
|
||||
|
||||
|
||||
def schedule_build(name, hour, minute=0):
|
||||
for current_branch in NIGHT_SCHEDULE_BRANCHES:
|
||||
scheduler_name = "nightly " + name
|
||||
if current_branch:
|
||||
scheduler_name += ' ' + current_branch
|
||||
# Use special addons submodule branch when building blender2.8 branch.
|
||||
addons_branch = "master"
|
||||
if current_branch == "blender2.8":
|
||||
addons_branch = "blender2.8"
|
||||
c['schedulers'].append(timed.Nightly(name=scheduler_name,
|
||||
codebases={
|
||||
"blender": {"repository": ""},
|
||||
"blender-translations": {"repository": "", "branch": "master"},
|
||||
"blender-addons": {"repository": "", "branch": addons_branch},
|
||||
"blender-addons-contrib": {"repository": "", "branch": "master"},
|
||||
"blender-dev-tools": {"repository": "", "branch": "master"},
|
||||
"lib svn": {"repository": "", "branch": "trunk"}},
|
||||
branch=current_branch,
|
||||
builderNames=[name],
|
||||
hour=hour,
|
||||
minute=minute))
|
||||
|
||||
|
||||
# BUILDERS
|
||||
#
|
||||
# The 'builders' list defines the Builders, which tell Buildbot how to
|
||||
# perform a build: what steps, and which slaves can execute them.
|
||||
# Note that any particular build will only take place on one slave.
|
||||
|
||||
from buildbot.process.factory import BuildFactory
|
||||
from buildbot.process.properties import Interpolate
|
||||
from buildbot.steps.source import SVN
|
||||
from buildbot.steps.source import Git
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from buildbot.steps.shell import Compile
|
||||
from buildbot.steps.shell import Test
|
||||
from buildbot.steps.transfer import FileUpload
|
||||
from buildbot.steps.master import MasterShellCommand
|
||||
from buildbot.config import BuilderConfig
|
||||
|
||||
# add builder utility
|
||||
|
||||
c['builders'] = []
|
||||
buildernames = []
|
||||
|
||||
|
||||
def add_builder(c, name, libdir, factory, branch='',
|
||||
rsync=False, hour=3, minute=0):
|
||||
slavenames = []
|
||||
|
||||
for slave in master_private.slaves:
|
||||
if name in slave['builders']:
|
||||
slavenames.append(slave['name'])
|
||||
|
||||
if len(slavenames) > 0:
|
||||
f = factory(name, libdir, branch, rsync)
|
||||
c['builders'].append(BuilderConfig(name=name,
|
||||
slavenames=slavenames,
|
||||
factory=f,
|
||||
category='blender'))
|
||||
buildernames.append(name)
|
||||
|
||||
schedule_build(name, hour, minute)
|
||||
schedule_force_build(name)
|
||||
|
||||
# common steps
|
||||
|
||||
|
||||
def git_submodule_step(submodule):
|
||||
return Git(name=submodule + '.git',
|
||||
repourl='git://git.blender.org/' + submodule + '.git',
|
||||
mode='update',
|
||||
codebase=submodule,
|
||||
workdir=submodule + '.git')
|
||||
|
||||
|
||||
def git_step(branch=''):
|
||||
if branch:
|
||||
return Git(name='blender.git',
|
||||
repourl='git://git.blender.org/blender.git',
|
||||
mode='update',
|
||||
branch=branch,
|
||||
codebase='blender',
|
||||
workdir='blender.git',
|
||||
submodules=True)
|
||||
else:
|
||||
return Git(name='blender.git',
|
||||
repourl='git://git.blender.org/blender.git',
|
||||
mode='update',
|
||||
codebase='blender',
|
||||
workdir='blender.git',
|
||||
submodules=True)
|
||||
|
||||
|
||||
def git_submodules_update():
|
||||
command = ['git', 'submodule', 'update', '--remote']
|
||||
return ShellCommand(name='Submodules Update',
|
||||
command=command,
|
||||
description='updating',
|
||||
descriptionDone='up to date',
|
||||
workdir='blender.git')
|
||||
|
||||
|
||||
def lib_svn_step(dir):
|
||||
return SVN(name='lib svn',
|
||||
baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
|
||||
codebase='lib svn',
|
||||
mode='update',
|
||||
defaultBranch='trunk',
|
||||
workdir='lib/' + dir)
|
||||
|
||||
|
||||
def rsync_step(id, branch, rsync_script):
|
||||
return ShellCommand(name='rsync',
|
||||
command=['python', rsync_script, id, branch],
|
||||
description='uploading',
|
||||
descriptionDone='uploaded',
|
||||
workdir='install')
|
||||
|
||||
# generic builder
|
||||
|
||||
|
||||
def generic_builder(id, libdir='', branch='', rsync=False):
|
||||
filename = 'uploaded/buildbot_upload_' + id + '.zip'
|
||||
compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
|
||||
test_script = '../blender.git/build_files/buildbot/slave_test.py'
|
||||
pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
|
||||
rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
|
||||
unpack_script = 'master_unpack.py'
|
||||
|
||||
f = BuildFactory()
|
||||
if libdir != '':
|
||||
f.addStep(lib_svn_step(libdir))
|
||||
|
||||
for submodule in ('blender-translations',
|
||||
'blender-addons',
|
||||
'blender-addons-contrib',
|
||||
'blender-dev-tools'):
|
||||
f.addStep(git_submodule_step(submodule))
|
||||
|
||||
f.addStep(git_step(branch))
|
||||
f.addStep(git_submodules_update())
|
||||
|
||||
f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
|
||||
f.addStep(Test(command=['python', test_script, id]))
|
||||
f.addStep(ShellCommand(name='package',
|
||||
command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
|
||||
description='packaging',
|
||||
descriptionDone='packaged'))
|
||||
if rsync:
|
||||
f.addStep(rsync_step(id, branch, rsync_script))
|
||||
else:
|
||||
f.addStep(FileUpload(name='upload',
|
||||
slavesrc='buildbot_upload.zip',
|
||||
masterdest=filename,
|
||||
maxsize=180 * 1024 * 1024,
|
||||
workdir='install'))
|
||||
f.addStep(MasterShellCommand(name='unpack',
|
||||
command=['python2.7', unpack_script, filename],
|
||||
description='unpacking',
|
||||
descriptionDone='unpacked'))
|
||||
return f
|
||||
|
||||
# Builders
|
||||
|
||||
add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
|
||||
add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2)
|
||||
add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1)
|
||||
add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1)
|
||||
add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2)
|
||||
add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3)
|
||||
add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4)
|
||||
|
||||
# STATUS TARGETS
|
||||
#
|
||||
# 'status' is a list of Status Targets. The results of each build will be
|
||||
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
|
||||
# including web pages, email senders, and IRC bots.
|
||||
|
||||
c['status'] = []
|
||||
|
||||
from buildbot.status import html
|
||||
from buildbot.status.web import authz
|
||||
from buildbot.status.web import auth
|
||||
|
||||
users = []
|
||||
for slave in master_private.slaves:
|
||||
users += [(slave['name'], slave['password'])]
|
||||
|
||||
authz_cfg = authz.Authz(
|
||||
auth=auth.BasicAuth(users),
|
||||
# change any of these to True to enable; see the manual for more
|
||||
# options
|
||||
gracefulShutdown=False,
|
||||
forceBuild=True, # use this to test your slave once it is set up
|
||||
forceAllBuilds=False,
|
||||
pingBuilder=False,
|
||||
stopBuild=True,
|
||||
stopAllBuilds=False,
|
||||
cancelPendingBuild=True,
|
||||
)
|
||||
|
||||
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
||||
#c['status'].append(html.WebStatus(http_port=8010))
|
||||
|
||||
# PROJECT IDENTITY
|
||||
|
||||
c['projectName'] = "Blender"
|
||||
c['projectURL'] = "http://www.blender.org"
|
||||
|
||||
# the 'buildbotURL' string should point to the location where the buildbot's
|
||||
# internal web server (usually the html.WebStatus page) is visible. This
|
||||
# typically uses the port number set in the Waterfall 'status' entry, but
|
||||
# with an externally-visible host name which the buildbot cannot figure out
|
||||
# without some help.
|
||||
|
||||
c['buildbotURL'] = "http://builder.blender.org/"
|
||||
|
||||
# DB URL
|
||||
#
|
||||
# This specifies what database buildbot uses to store change and scheduler
|
||||
# state. You can leave this at its default for all but the largest
|
||||
# installations.
|
||||
|
||||
c['db_url'] = "sqlite:///state.sqlite"
|
||||
@ -1,151 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# Runs on Buildbot master, to unpack incoming unload.zip into latest
|
||||
# builds directory and remove older builds.
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
# extension stripping
|
||||
def strip_extension(filename):
|
||||
extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'
|
||||
|
||||
for ext in extensions:
|
||||
if filename.endswith(ext):
|
||||
filename = filename[:-len(ext)]
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
# extract platform from package name
|
||||
def get_platform(filename):
|
||||
# name is blender-version-platform.extension. we want to get the
|
||||
# platform out, but there may be some variations, so we fiddle a
|
||||
# bit to handle current and hopefully future names
|
||||
filename = strip_extension(filename)
|
||||
filename = strip_extension(filename)
|
||||
|
||||
tokens = filename.split("-")
|
||||
platforms = ('osx', 'mac', 'bsd',
|
||||
'win', 'linux', 'source',
|
||||
'irix', 'solaris')
|
||||
platform_tokens = []
|
||||
found = False
|
||||
|
||||
for i, token in enumerate(tokens):
|
||||
if not found:
|
||||
for platform in platforms:
|
||||
if platform in token.lower():
|
||||
found = True
|
||||
break
|
||||
|
||||
if found:
|
||||
platform_tokens += [token]
|
||||
|
||||
return '-'.join(platform_tokens)
|
||||
|
||||
|
||||
def get_branch(filename):
|
||||
if filename.startswith("blender-2.8"):
|
||||
return "blender2.8"
|
||||
|
||||
tokens = filename.split("-")
|
||||
branch = ""
|
||||
|
||||
for token in tokens:
|
||||
if token == "blender":
|
||||
return branch
|
||||
|
||||
if branch == "":
|
||||
branch = token
|
||||
else:
|
||||
branch = branch + "-" + token
|
||||
|
||||
return ""
|
||||
|
||||
# get filename
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting file to unpack\n")
|
||||
sys.exit(1)
|
||||
|
||||
filename = sys.argv[1]
|
||||
|
||||
# open zip file
|
||||
if not os.path.exists(filename):
|
||||
sys.stderr.write("File %r not found.\n" % filename)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
z = zipfile.ZipFile(filename, "r")
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Failed to open zip file: %s\n' % str(ex))
|
||||
sys.exit(1)
|
||||
|
||||
if len(z.namelist()) != 1:
|
||||
sys.stderr.write("Expected one file in %r." % filename)
|
||||
sys.exit(1)
|
||||
|
||||
package = z.namelist()[0]
|
||||
packagename = os.path.basename(package)
|
||||
|
||||
# detect platform and branch
|
||||
platform = get_platform(packagename)
|
||||
branch = get_branch(packagename)
|
||||
|
||||
if platform == '':
|
||||
sys.stderr.write('Failed to detect platform ' +
|
||||
'from package: %r\n' % packagename)
|
||||
sys.exit(1)
|
||||
|
||||
# extract
|
||||
if not branch or branch == 'master':
|
||||
directory = 'public_html/download'
|
||||
elif branch == 'experimental-build':
|
||||
directory = 'public_html/download/experimental'
|
||||
else:
|
||||
directory = 'public_html/download'
|
||||
|
||||
try:
|
||||
filename = os.path.join(directory, packagename)
|
||||
zf = z.open(package)
|
||||
f = file(filename, "wb")
|
||||
|
||||
shutil.copyfileobj(zf, f)
|
||||
os.chmod(filename, 0644)
|
||||
|
||||
zf.close()
|
||||
z.close()
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Failed to unzip package: %s\n' % str(ex))
|
||||
sys.exit(1)
|
||||
|
||||
# remove other files from the same platform and branch
|
||||
try:
|
||||
for f in os.listdir(directory):
|
||||
if get_platform(f) == platform and get_branch(f) == branch:
|
||||
if f != packagename:
|
||||
os.remove(os.path.join(directory, f))
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Failed to remove old packages: %s\n' % str(ex))
|
||||
sys.exit(1)
|
||||
Loading…
Reference in New Issue
Block a user