Variety of fixes all over the place
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "gooseberry"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "gooseberry", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to controll separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'git://git.blender.org/scons.git': 'scons',
98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
99 }
100
101
102 def codebaseGenerator(chdict):
103     return all_repositories[chdict['repository']]
104
105 c['codebaseGenerator'] = codebaseGenerator
106
107
108 # SCHEDULERS
109 #
110 # Decide how to react to incoming changes.
111
112 # from buildbot.scheduler import Scheduler
113 from buildbot.schedulers import timed, forcesched
114
115 c['schedulers'] = []
116
117
118 def schedule_force_build(name):
119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
120         builderNames=[name],
121         codebases=[forcesched.CodebaseParameter(
122                 codebase="blender",
123                 branch=forcesched.ChoiceStringParameter(
124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
125                 # Do not hide revision, can be handy!
126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
128             # For now, hide other codebases.
129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
130             forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
131             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
132             forcesched.CodebaseParameter(hide=True, codebase="scons"),
133             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
134         properties=[]))
135
136
137 def schedule_build(name, hour, minute=0):
138     for current_branch in NIGHT_SCHEDULE_BRANCHES:
139         scheduler_name = "nightly " + name
140         if current_branch:
141             scheduler_name += ' ' + current_branch
142         c['schedulers'].append(timed.Nightly(name=scheduler_name,
143             codebases={
144                 "blender": {"repository": ""},
145                 "blender-translations": {"repository": "", "branch": "master"},
146                 "blender-addons": {"repository": "", "branch": "master"},
147                 "blender-addons-contrib": {"repository": "", "branch": "master"},
148                 "scons": {"repository": "", "branch": "master"},
149                 "lib svn": {"repository": "", "branch": "trunk"}},
150             branch=current_branch,
151             builderNames=[name],
152             hour=hour,
153             minute=minute))
154
155
156 # BUILDERS
157 #
158 # The 'builders' list defines the Builders, which tell Buildbot how to
159 # perform a build: what steps, and which slaves can execute them.
160 # Note that any particular build will only take place on one slave.
161
162 from buildbot.process.factory import BuildFactory
163 from buildbot.process.properties import Interpolate
164 from buildbot.steps.source import SVN
165 from buildbot.steps.source import Git
166 from buildbot.steps.shell import ShellCommand
167 from buildbot.steps.shell import Compile
168 from buildbot.steps.shell import Test
169 from buildbot.steps.transfer import FileUpload
170 from buildbot.steps.master import MasterShellCommand
171 from buildbot.config import BuilderConfig
172
173 # add builder utility
174
175 c['builders'] = []
176 buildernames = []
177
178
179 def add_builder(c, name, libdir, factory, branch='',
180                 rsync=False, hour=3, minute=0):
181     slavenames = []
182
183     for slave in master_private.slaves:
184         if name in slave['builders']:
185             slavenames.append(slave['name'])
186
187     if len(slavenames) > 0:
188         f = factory(name, libdir, branch, rsync)
189         c['builders'].append(BuilderConfig(name=name,
190                                            slavenames=slavenames,
191                                            factory=f,
192                                            category='blender'))
193         buildernames.append(name)
194
195         schedule_build(name, hour, minute)
196         schedule_force_build(name)
197
198 # common steps
199
200
201 def git_submodule_step(submodule):
202     return Git(name=submodule + '.git',
203                repourl='git://git.blender.org/' + submodule + '.git',
204                mode='update',
205                codebase=submodule,
206                workdir=submodule + '.git')
207
208
209 def git_step(branch=''):
210     if branch:
211         return Git(name='blender.git',
212                    repourl='git://git.blender.org/blender.git',
213                    mode='update',
214                    branch=branch,
215                    codebase='blender',
216                    workdir='blender.git',
217                    submodules=True)
218     else:
219         return Git(name='blender.git',
220                    repourl='git://git.blender.org/blender.git',
221                    mode='update',
222                    codebase='blender',
223                    workdir='blender.git',
224                    submodules=True)
225
226
227 def git_submodules_update():
228     command = ['git', 'submodule', 'foreach', '--recursive',
229                'git', 'pull', 'origin', 'master']
230     return ShellCommand(name='Submodules Update',
231                         command=command,
232                         description='updating',
233                         descriptionDone='up to date',
234                         workdir='blender.git')
235
236
237 def lib_svn_step(dir):
238     return SVN(name='lib svn',
239                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
240                codebase='lib svn',
241                mode='update',
242                defaultBranch='trunk',
243                workdir='lib/' + dir)
244
245
246 def rsync_step(id, branch, rsync_script):
247     return ShellCommand(name='rsync',
248                         command=['python', rsync_script, id, branch],
249                         description='uploading',
250                         descriptionDone='uploaded',
251                         workdir='install')
252
253 # generic builder
254
255
256 def generic_builder(id, libdir='', branch='', rsync=False):
257     filename = 'uploaded/buildbot_upload_' + id + '.zip'
258     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
259     test_script = '../blender.git/build_files/buildbot/slave_test.py'
260     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
261     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
262     unpack_script = 'master_unpack.py'
263
264     f = BuildFactory()
265     if libdir != '':
266         f.addStep(lib_svn_step(libdir))
267
268     for submodule in ('blender-translations',
269                       'blender-addons',
270                       'blender-addons-contrib',
271                       'scons'):
272         f.addStep(git_submodule_step(submodule))
273
274     f.addStep(git_step(branch))
275     f.addStep(git_submodules_update())
276
277     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
278     f.addStep(Test(command=['python', test_script, id]))
279     f.addStep(ShellCommand(name='package',
280                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
281                            description='packaging',
282                            descriptionDone='packaged'))
283     if rsync:
284         f.addStep(rsync_step(id, branch, rsync_script))
285     elif id.find('cmake') != -1:
286         f.addStep(FileUpload(name='upload',
287                              slavesrc='buildbot_upload.zip',
288                              masterdest=filename,
289                              maxsize=150 * 1024 * 1024))
290     else:
291         f.addStep(FileUpload(name='upload',
292                              slavesrc='buildbot_upload.zip',
293                              masterdest=filename,
294                              maxsize=150 * 1024 * 1024,
295                              workdir='install'))
296     f.addStep(MasterShellCommand(name='unpack',
297                                  command=['python', unpack_script, filename],
298                                  description='unpacking',
299                                  descriptionDone='unpacked'))
300     return f
301
302 # builders
303
304 add_builder(c, 'mac_x86_64_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=5)
305 add_builder(c, 'mac_i386_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=11)
306 add_builder(c, 'linux_glibc211_i386_scons', '', generic_builder, hour=1)
307 add_builder(c, 'linux_glibc211_x86_64_scons', '', generic_builder, hour=2)
308 add_builder(c, 'win32_scons_vc2013', 'windows_vc12', generic_builder, hour=1)
309 add_builder(c, 'win64_scons_vc2013', 'win64_vc12', generic_builder, hour=2)
310 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
311 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
312 #add_builder(c, 'mingw_win32_scons', 'mingw32', generic_builder, hour=4)
313 add_builder(c, 'mingw_win64_scons', 'mingw64', generic_builder, hour=3)
314 #add_builder(c, 'freebsd_i386_cmake', '', generic_builder, hour=1)
315 #add_builder(c, 'freebsd_x86_64_cmake', '', generic_builder, hour=2)
316
317 # Multiview branch
318 add_builder(c, 'multiview_win64_scons', 'win64', generic_builder, 'multiview', hour=4)
319 add_builder(c, 'multiview_win32_scons', 'windows', generic_builder, 'multiview', hour=5)
320
321 # STATUS TARGETS
322 #
323 # 'status' is a list of Status Targets. The results of each build will be
324 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
325 # including web pages, email senders, and IRC bots.
326
327 c['status'] = []
328
329 from buildbot.status import html
330 from buildbot.status.web import authz
331 from buildbot.status.web import auth
332
333 users = []
334 for slave in master_private.slaves:
335     users += [(slave['name'], slave['password'])]
336
337 authz_cfg = authz.Authz(
338     auth=auth.BasicAuth(users),
339     # change any of these to True to enable; see the manual for more
340     # options
341     gracefulShutdown=False,
342     forceBuild=True,  # use this to test your slave once it is set up
343     forceAllBuilds=False,
344     pingBuilder=False,
345     stopBuild=True,
346     stopAllBuilds=False,
347     cancelPendingBuild=True,
348 )
349
350 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
351 #c['status'].append(html.WebStatus(http_port=8010))
352
353 # PROJECT IDENTITY
354
355 c['projectName'] = "Blender"
356 c['projectURL'] = "http://www.blender.org"
357
358 # the 'buildbotURL' string should point to the location where the buildbot's
359 # internal web server (usually the html.WebStatus page) is visible. This
360 # typically uses the port number set in the Waterfall 'status' entry, but
361 # with an externally-visible host name which the buildbot cannot figure out
362 # without some help.
363
364 c['buildbotURL'] = "http://builder.blender.org/"
365
366 # DB URL
367 #
368 # This specifies what database buildbot uses to store change and scheduler
369 # state.  You can leave this at its default for all but the largest
370 # installations.
371
372 c['db_url'] = "sqlite:///state.sqlite"