build_files/buildbot/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 # <pep8 compliant>
   5
   6 # List of the branches being built automatically overnight
   7 NIGHT_SCHEDULE_BRANCHES = [None]
   8
   9 # List of the branches available for force build
  10 FORCE_SCHEDULE_BRANCHES = ["master", "gooseberry", "experimental-build"]
  11
  12 """
  13 Stock Twisted directory lister doesn't provide any information about last file
  14 modification time, we hack the class a bit in order to have such functionaliity
  15 :)
  16 """
  17
  18 from buildbot.status.web.base import DirectoryLister
  19
  20
  21 def get_files_and_directories(self, directory):
  22     from twisted.web.static import (getTypeAndEncoding,
  23                                     formatFileSize)
  24     import urllib
  25     import cgi
  26     import time
  27     import os
  28     files = []
  29     dirs = []
  30     for path in directory:
  31         url = urllib.quote(path, "/")
  32         escapedPath = cgi.escape(path)
  33         lastmodified = time.ctime(os.path.getmtime(
  34             os.path.join(self.path, path)))
  35         if os.path.isdir(os.path.join(self.path, path)):
  36             url = url + '/'
  37             dirs.append({'text': escapedPath + "/", 'href': url,
  38                          'size': '', 'type': '[Directory]',
  39                          'encoding': '',
  40                          'lastmodified': lastmodified})
  41         else:
  42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
  43                                                     self.contentEncodings,
  44                                                     self.defaultType)
  45             try:
  46                 size = os.stat(os.path.join(self.path, path)).st_size
  47             except OSError:
  48                 continue
  49             files.append({
  50                 'text': escapedPath, "href": url,
  51                 'type': '[%s]' % mimetype,
  52                 'encoding': (encoding and '[%s]' % encoding or ''),
  53                 'size': formatFileSize(size),
  54                 'lastmodified': lastmodified})
  55     return dirs, files
  56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
  57
  58 # Dictionary that the buildmaster pays attention to.
  59 c = BuildmasterConfig = {}
  60
  61 # BUILD SLAVES
  62 #
  63 # We load the slaves and their passwords from a separator file, so we can have
  64 # this one in SVN.
  65
  66 from buildbot.buildslave import BuildSlave
  67 import master_private
  68
  69 c['slaves'] = []
  70
  71 for slave in master_private.slaves:
  72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
  73
  74 # TCP port through which slaves connect
  75
  76 c['slavePortnum'] = 9989
  77
  78 # CHANGE SOURCES
  79
  80 from buildbot.changes.svnpoller import SVNPoller
  81 from buildbot.changes.gitpoller import GitPoller
  82
  83 c['change_source'] = GitPoller(
  84        'git://git.blender.org/blender.git',
  85        pollinterval=1200)
  86
  87
  88 # CODEBASES
  89 #
  90 # Allow to controll separately things like branches for each repo and submodules.
  91
  92 all_repositories = {
  93     r'git://git.blender.org/blender.git': 'blender',
  94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
  95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
  96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
  97     r'git://git.blender.org/scons.git': 'scons',
  98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
  99 }
 100
 101
 102 def codebaseGenerator(chdict):
 103     return all_repositories[chdict['repository']]
 104
 105 c['codebaseGenerator'] = codebaseGenerator
 106
 107
 108 # SCHEDULERS
 109 #
 110 # Decide how to react to incoming changes.
 111
 112 # from buildbot.scheduler import Scheduler
 113 from buildbot.schedulers import timed, forcesched
 114
 115 c['schedulers'] = []
 116
 117
 118 def schedule_force_build(name):
 119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
 120         builderNames=[name],
 121         codebases=[forcesched.CodebaseParameter(
 122                 codebase="blender",
 123                 branch=forcesched.ChoiceStringParameter(
 124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
 125                 # Do not hide revision, can be handy!
 126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
 127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
 128             # For now, hide other codebases.
 129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
 130             forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
 131             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
 132             forcesched.CodebaseParameter(hide=True, codebase="scons"),
 133             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
 134         properties=[]))
 135
 136
 137 def schedule_build(name, hour, minute=0):
 138     for current_branch in NIGHT_SCHEDULE_BRANCHES:
 139         scheduler_name = "nightly " + name
 140         if current_branch:
 141             scheduler_name += ' ' + current_branch
 142         c['schedulers'].append(timed.Nightly(name=scheduler_name,
 143             codebases={
 144                 "blender": {"repository": ""},
 145                 "blender-translations": {"repository": "", "branch": "master"},
 146                 "blender-addons": {"repository": "", "branch": "master"},
 147                 "blender-addons-contrib": {"repository": "", "branch": "master"},
 148                 "scons": {"repository": "", "branch": "master"},
 149                 "lib svn": {"repository": "", "branch": "trunk"}},
 150             branch=current_branch,
 151             builderNames=[name],
 152             hour=hour,
 153             minute=minute))
 154
 155
 156 # BUILDERS
 157 #
 158 # The 'builders' list defines the Builders, which tell Buildbot how to
 159 # perform a build: what steps, and which slaves can execute them.
 160 # Note that any particular build will only take place on one slave.
 161
 162 from buildbot.process.factory import BuildFactory
 163 from buildbot.process.properties import Interpolate
 164 from buildbot.steps.source import SVN
 165 from buildbot.steps.source import Git
 166 from buildbot.steps.shell import ShellCommand
 167 from buildbot.steps.shell import Compile
 168 from buildbot.steps.shell import Test
 169 from buildbot.steps.transfer import FileUpload
 170 from buildbot.steps.master import MasterShellCommand
 171 from buildbot.config import BuilderConfig
 172
 173 # add builder utility
 174
 175 c['builders'] = []
 176 buildernames = []
 177
 178
 179 def add_builder(c, name, libdir, factory, branch='',
 180                 rsync=False, hour=3, minute=0):
 181     slavenames = []
 182
 183     for slave in master_private.slaves:
 184         if name in slave['builders']:
 185             slavenames.append(slave['name'])
 186
 187     if len(slavenames) > 0:
 188         f = factory(name, libdir, branch, rsync)
 189         c['builders'].append(BuilderConfig(name=name,
 190                                            slavenames=slavenames,
 191                                            factory=f,
 192                                            category='blender'))
 193         buildernames.append(name)
 194
 195         schedule_build(name, hour, minute)
 196         schedule_force_build(name)
 197
 198 # common steps
 199
 200
 201 def git_submodule_step(submodule):
 202     return Git(name=submodule + '.git',
 203                repourl='git://git.blender.org/' + submodule + '.git',
 204                mode='update',
 205                codebase=submodule,
 206                workdir=submodule + '.git')
 207
 208
 209 def git_step(branch=''):
 210     if branch:
 211         return Git(name='blender.git',
 212                    repourl='git://git.blender.org/blender.git',
 213                    mode='update',
 214                    branch=branch,
 215                    codebase='blender',
 216                    workdir='blender.git',
 217                    submodules=True)
 218     else:
 219         return Git(name='blender.git',
 220                    repourl='git://git.blender.org/blender.git',
 221                    mode='update',
 222                    codebase='blender',
 223                    workdir='blender.git',
 224                    submodules=True)
 225
 226
 227 def git_submodules_update():
 228     command = ['git', 'submodule', 'foreach', '--recursive',
 229                'git', 'pull', 'origin', 'master']
 230     return ShellCommand(name='Submodules Update',
 231                         command=command,
 232                         description='updating',
 233                         descriptionDone='up to date',
 234                         workdir='blender.git')
 235
 236
 237 def lib_svn_step(dir):
 238     return SVN(name='lib svn',
 239                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
 240                codebase='lib svn',
 241                mode='update',
 242                defaultBranch='trunk',
 243                workdir='lib/' + dir)
 244
 245
 246 def rsync_step(id, branch, rsync_script):
 247     return ShellCommand(name='rsync',
 248                         command=['python', rsync_script, id, branch],
 249                         description='uploading',
 250                         descriptionDone='uploaded',
 251                         workdir='install')
 252
 253 # generic builder
 254
 255
 256 def generic_builder(id, libdir='', branch='', rsync=False):
 257     filename = 'uploaded/buildbot_upload_' + id + '.zip'
 258     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
 259     test_script = '../blender.git/build_files/buildbot/slave_test.py'
 260     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
 261     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
 262     unpack_script = 'master_unpack.py'
 263
 264     f = BuildFactory()
 265     if libdir != '':
 266         f.addStep(lib_svn_step(libdir))
 267
 268     for submodule in ('blender-translations',
 269                       'blender-addons',
 270                       'blender-addons-contrib',
 271                       'scons'):
 272         f.addStep(git_submodule_step(submodule))
 273
 274     f.addStep(git_step(branch))
 275     f.addStep(git_submodules_update())
 276
 277     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
 278     f.addStep(Test(command=['python', test_script, id]))
 279     f.addStep(ShellCommand(name='package',
 280                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
 281                            description='packaging',
 282                            descriptionDone='packaged'))
 283     if rsync:
 284         f.addStep(rsync_step(id, branch, rsync_script))
 285     else:
 286         f.addStep(FileUpload(name='upload',
 287                              slavesrc='buildbot_upload.zip',
 288                              masterdest=filename,
 289                              maxsize=150 * 1024 * 1024,
 290                              workdir='install'))
 291     f.addStep(MasterShellCommand(name='unpack',
 292                                  command=['python', unpack_script, filename],
 293                                  description='unpacking',
 294                                  descriptionDone='unpacked'))
 295     return f
 296
 297 # Builders
 298
 299 add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
 300 add_builder(c, 'mac_i386_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=11)
 301 add_builder(c, 'linux_glibc211_i386_cmake', '', generic_builder, hour=1)
 302 add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
 303 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
 304 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
 305
 306 # STATUS TARGETS
 307 #
 308 # 'status' is a list of Status Targets. The results of each build will be
 309 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 310 # including web pages, email senders, and IRC bots.
 311
 312 c['status'] = []
 313
 314 from buildbot.status import html
 315 from buildbot.status.web import authz
 316 from buildbot.status.web import auth
 317
 318 users = []
 319 for slave in master_private.slaves:
 320     users += [(slave['name'], slave['password'])]
 321
 322 authz_cfg = authz.Authz(
 323     auth=auth.BasicAuth(users),
 324     # change any of these to True to enable; see the manual for more
 325     # options
 326     gracefulShutdown=False,
 327     forceBuild=True,  # use this to test your slave once it is set up
 328     forceAllBuilds=False,
 329     pingBuilder=False,
 330     stopBuild=True,
 331     stopAllBuilds=False,
 332     cancelPendingBuild=True,
 333 )
 334
 335 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
 336 #c['status'].append(html.WebStatus(http_port=8010))
 337
 338 # PROJECT IDENTITY
 339
 340 c['projectName'] = "Blender"
 341 c['projectURL'] = "http://www.blender.org"
 342
 343 # the 'buildbotURL' string should point to the location where the buildbot's
 344 # internal web server (usually the html.WebStatus page) is visible. This
 345 # typically uses the port number set in the Waterfall 'status' entry, but
 346 # with an externally-visible host name which the buildbot cannot figure out
 347 # without some help.
 348
 349 c['buildbotURL'] = "http://builder.blender.org/"
 350
 351 # DB URL
 352 #
 353 # This specifies what database buildbot uses to store change and scheduler
 354 # state.  You can leave this at its default for all but the largest
 355 # installations.
 356
 357 c['db_url'] = "sqlite:///state.sqlite"