Use more meaningful name for C++11 libs step
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to control separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
99 }
100
101
102 def codebaseGenerator(chdict):
103     return all_repositories[chdict['repository']]
104
105 c['codebaseGenerator'] = codebaseGenerator
106
107
108 # SCHEDULERS
109 #
110 # Decide how to react to incoming changes.
111
112 # from buildbot.scheduler import Scheduler
113 from buildbot.schedulers import timed, forcesched
114
115 c['schedulers'] = []
116
117
118 def schedule_force_build(name):
119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
120         builderNames=[name],
121         codebases=[forcesched.CodebaseParameter(
122                 codebase="blender",
123                 branch=forcesched.ChoiceStringParameter(
124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
125                 # Do not hide revision, can be handy!
126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
128             # For now, hide other codebases.
129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
130             forcesched.CodebaseParameter(
131                 codebase="blender-addons",
132                 branch=forcesched.ChoiceStringParameter(
133                     name="branch", choices=["master", "blender2.8"], default="master"),
134                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
135                 project=forcesched.FixedParameter(name="project", default="", hide=True),
136                 revision=forcesched.FixedParameter(name="revision", default="", hide=True),
137             ),
138             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
139             forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
140             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
141         properties=[]))
142
143
144 def schedule_build(name, hour, minute=0):
145     for current_branch in NIGHT_SCHEDULE_BRANCHES:
146         scheduler_name = "nightly " + name
147         if current_branch:
148             scheduler_name += ' ' + current_branch
149         # Use special addons submodule branch when building blender2.8 branch.
150         addons_branch = "master"
151         if current_branch == "blender2.8":
152             addons_branch = "blender2.8"
153         c['schedulers'].append(timed.Nightly(name=scheduler_name,
154             codebases={
155                 "blender": {"repository": ""},
156                 "blender-translations": {"repository": "", "branch": "master"},
157                 "blender-addons": {"repository": "", "branch": "master"},
158                 "blender-addons-contrib": {"repository": "", "branch": addons_branch},
159                 "blender-dev-tools": {"repository": "", "branch": "master"},
160                 "lib svn": {"repository": "", "branch": "trunk"}},
161             branch=current_branch,
162             builderNames=[name],
163             hour=hour,
164             minute=minute))
165
166
167 # BUILDERS
168 #
169 # The 'builders' list defines the Builders, which tell Buildbot how to
170 # perform a build: what steps, and which slaves can execute them.
171 # Note that any particular build will only take place on one slave.
172
173 from buildbot.process.factory import BuildFactory
174 from buildbot.process.properties import Interpolate
175 from buildbot.steps.source import SVN
176 from buildbot.steps.source import Git
177 from buildbot.steps.shell import ShellCommand
178 from buildbot.steps.shell import Compile
179 from buildbot.steps.shell import Test
180 from buildbot.steps.transfer import FileUpload
181 from buildbot.steps.master import MasterShellCommand
182 from buildbot.config import BuilderConfig
183
184 # add builder utility
185
186 c['builders'] = []
187 buildernames = []
188
189
190 def add_builder(c, name, libdir, factory, branch='',
191                 rsync=False, hour=3, minute=0):
192     slavenames = []
193
194     for slave in master_private.slaves:
195         if name in slave['builders']:
196             slavenames.append(slave['name'])
197
198     if len(slavenames) > 0:
199         f = factory(name, libdir, branch, rsync)
200         c['builders'].append(BuilderConfig(name=name,
201                                            slavenames=slavenames,
202                                            factory=f,
203                                            category='blender'))
204         buildernames.append(name)
205
206         schedule_build(name, hour, minute)
207         schedule_force_build(name)
208
209 # common steps
210
211
212 def git_submodule_step(submodule):
213     return Git(name=submodule + '.git',
214                repourl='git://git.blender.org/' + submodule + '.git',
215                mode='update',
216                codebase=submodule,
217                workdir=submodule + '.git')
218
219
220 def git_step(branch=''):
221     if branch:
222         return Git(name='blender.git',
223                    repourl='git://git.blender.org/blender.git',
224                    mode='update',
225                    branch=branch,
226                    codebase='blender',
227                    workdir='blender.git',
228                    submodules=True)
229     else:
230         return Git(name='blender.git',
231                    repourl='git://git.blender.org/blender.git',
232                    mode='update',
233                    codebase='blender',
234                    workdir='blender.git',
235                    submodules=True)
236
237
238 def git_submodules_update():
239     command = ['git', 'submodule', 'update', '--remote']
240     return ShellCommand(name='Submodules Update',
241                         command=command,
242                         description='updating',
243                         descriptionDone='up to date',
244                         workdir='blender.git')
245
246
247 def lib_svn_step(dir):
248     name = "lib svn"
249     if dir == "darwin":
250         name = "C++11 lib svn"
251     return SVN(name=name,
252                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
253                codebase='lib svn',
254                mode='update',
255                defaultBranch='trunk',
256                workdir='lib/' + dir)
257
258
259 def rsync_step(id, branch, rsync_script):
260     return ShellCommand(name='rsync',
261                         command=['python', rsync_script, id, branch],
262                         description='uploading',
263                         descriptionDone='uploaded',
264                         workdir='install')
265
266 # generic builder
267
268
269 def generic_builder(id, libdir='', branch='', rsync=False):
270     filename = 'uploaded/buildbot_upload_' + id + '.zip'
271     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
272     test_script = '../blender.git/build_files/buildbot/slave_test.py'
273     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
274     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
275     unpack_script = 'master_unpack.py'
276
277     f = BuildFactory()
278     if libdir != '':
279         f.addStep(lib_svn_step(libdir))
280         # Special trick to make sure we always have all the libs.
281         if libdir.startswith("darwin"):
282             f.addStep(lib_svn_step("darwin"))
283
284     for submodule in ('blender-translations',
285                       'blender-addons',
286                       'blender-addons-contrib',
287                       'blender-dev-tools'):
288         f.addStep(git_submodule_step(submodule))
289
290     f.addStep(git_step(branch))
291     f.addStep(git_submodules_update())
292
293     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
294     f.addStep(Test(command=['python', test_script, id]))
295     f.addStep(ShellCommand(name='package',
296                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
297                            description='packaging',
298                            descriptionDone='packaged'))
299     if rsync:
300         f.addStep(rsync_step(id, branch, rsync_script))
301     else:
302         f.addStep(FileUpload(name='upload',
303                              slavesrc='buildbot_upload.zip',
304                              masterdest=filename,
305                              maxsize=150 * 1024 * 1024,
306                              workdir='install'))
307     f.addStep(MasterShellCommand(name='unpack',
308                                  command=['python2.7', unpack_script, filename],
309                                  description='unpacking',
310                                  descriptionDone='unpacked'))
311     return f
312
313 # Builders
314
315 add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
316 # add_builder(c, 'linux_glibc211_i686_cmake', '', generic_builder, hour=1)
317 # add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
318 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=3)
319 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=4)
320 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
321 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
322 add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=5)
323 add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=6)
324
325 # STATUS TARGETS
326 #
327 # 'status' is a list of Status Targets. The results of each build will be
328 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
329 # including web pages, email senders, and IRC bots.
330
331 c['status'] = []
332
333 from buildbot.status import html
334 from buildbot.status.web import authz
335 from buildbot.status.web import auth
336
337 users = []
338 for slave in master_private.slaves:
339     users += [(slave['name'], slave['password'])]
340
341 authz_cfg = authz.Authz(
342     auth=auth.BasicAuth(users),
343     # change any of these to True to enable; see the manual for more
344     # options
345     gracefulShutdown=False,
346     forceBuild=True,  # use this to test your slave once it is set up
347     forceAllBuilds=False,
348     pingBuilder=False,
349     stopBuild=True,
350     stopAllBuilds=False,
351     cancelPendingBuild=True,
352 )
353
354 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
355 #c['status'].append(html.WebStatus(http_port=8010))
356
357 # PROJECT IDENTITY
358
359 c['projectName'] = "Blender"
360 c['projectURL'] = "http://www.blender.org"
361
362 # the 'buildbotURL' string should point to the location where the buildbot's
363 # internal web server (usually the html.WebStatus page) is visible. This
364 # typically uses the port number set in the Waterfall 'status' entry, but
365 # with an externally-visible host name which the buildbot cannot figure out
366 # without some help.
367
368 c['buildbotURL'] = "http://builder.blender.org/"
369
370 # DB URL
371 #
372 # This specifies what database buildbot uses to store change and scheduler
373 # state.  You can leave this at its default for all but the largest
374 # installations.
375
376 c['db_url'] = "sqlite:///state.sqlite"