Add C++11 libraries step for OSX
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to control separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
99 }
100
101
102 def codebaseGenerator(chdict):
103     return all_repositories[chdict['repository']]
104
105 c['codebaseGenerator'] = codebaseGenerator
106
107
108 # SCHEDULERS
109 #
110 # Decide how to react to incoming changes.
111
112 # from buildbot.scheduler import Scheduler
113 from buildbot.schedulers import timed, forcesched
114
115 c['schedulers'] = []
116
117
118 def schedule_force_build(name):
119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
120         builderNames=[name],
121         codebases=[forcesched.CodebaseParameter(
122                 codebase="blender",
123                 branch=forcesched.ChoiceStringParameter(
124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
125                 # Do not hide revision, can be handy!
126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
128             # For now, hide other codebases.
129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
130             forcesched.CodebaseParameter(
131                 codebase="blender-addons",
132                 branch=forcesched.ChoiceStringParameter(
133                     name="branch", choices=["master", "blender2.8"], default="master"),
134                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
135                 project=forcesched.FixedParameter(name="project", default="", hide=True),
136                 revision=forcesched.FixedParameter(name="revision", default="", hide=True),
137             ),
138             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
139             forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
140             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
141         properties=[]))
142
143
144 def schedule_build(name, hour, minute=0):
145     for current_branch in NIGHT_SCHEDULE_BRANCHES:
146         scheduler_name = "nightly " + name
147         if current_branch:
148             scheduler_name += ' ' + current_branch
149         # Use special addons submodule branch when building blender2.8 branch.
150         addons_branch = "master"
151         if current_branch == "blender2.8":
152             addons_branch = "blender2.8"
153         c['schedulers'].append(timed.Nightly(name=scheduler_name,
154             codebases={
155                 "blender": {"repository": ""},
156                 "blender-translations": {"repository": "", "branch": "master"},
157                 "blender-addons": {"repository": "", "branch": "master"},
158                 "blender-addons-contrib": {"repository": "", "branch": addons_branch},
159                 "blender-dev-tools": {"repository": "", "branch": "master"},
160                 "lib svn": {"repository": "", "branch": "trunk"}},
161             branch=current_branch,
162             builderNames=[name],
163             hour=hour,
164             minute=minute))
165
166
167 # BUILDERS
168 #
169 # The 'builders' list defines the Builders, which tell Buildbot how to
170 # perform a build: what steps, and which slaves can execute them.
171 # Note that any particular build will only take place on one slave.
172
173 from buildbot.process.factory import BuildFactory
174 from buildbot.process.properties import Interpolate
175 from buildbot.steps.source import SVN
176 from buildbot.steps.source import Git
177 from buildbot.steps.shell import ShellCommand
178 from buildbot.steps.shell import Compile
179 from buildbot.steps.shell import Test
180 from buildbot.steps.transfer import FileUpload
181 from buildbot.steps.master import MasterShellCommand
182 from buildbot.config import BuilderConfig
183
184 # add builder utility
185
186 c['builders'] = []
187 buildernames = []
188
189
190 def add_builder(c, name, libdir, factory, branch='',
191                 rsync=False, hour=3, minute=0):
192     slavenames = []
193
194     for slave in master_private.slaves:
195         if name in slave['builders']:
196             slavenames.append(slave['name'])
197
198     if len(slavenames) > 0:
199         f = factory(name, libdir, branch, rsync)
200         c['builders'].append(BuilderConfig(name=name,
201                                            slavenames=slavenames,
202                                            factory=f,
203                                            category='blender'))
204         buildernames.append(name)
205
206         schedule_build(name, hour, minute)
207         schedule_force_build(name)
208
209 # common steps
210
211
212 def git_submodule_step(submodule):
213     return Git(name=submodule + '.git',
214                repourl='git://git.blender.org/' + submodule + '.git',
215                mode='update',
216                codebase=submodule,
217                workdir=submodule + '.git')
218
219
220 def git_step(branch=''):
221     if branch:
222         return Git(name='blender.git',
223                    repourl='git://git.blender.org/blender.git',
224                    mode='update',
225                    branch=branch,
226                    codebase='blender',
227                    workdir='blender.git',
228                    submodules=True)
229     else:
230         return Git(name='blender.git',
231                    repourl='git://git.blender.org/blender.git',
232                    mode='update',
233                    codebase='blender',
234                    workdir='blender.git',
235                    submodules=True)
236
237
238 def git_submodules_update():
239     command = ['git', 'submodule', 'update', '--remote']
240     return ShellCommand(name='Submodules Update',
241                         command=command,
242                         description='updating',
243                         descriptionDone='up to date',
244                         workdir='blender.git')
245
246
247 def lib_svn_step(dir):
248     return SVN(name='lib svn',
249                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
250                codebase='lib svn',
251                mode='update',
252                defaultBranch='trunk',
253                workdir='lib/' + dir)
254
255
256 def rsync_step(id, branch, rsync_script):
257     return ShellCommand(name='rsync',
258                         command=['python', rsync_script, id, branch],
259                         description='uploading',
260                         descriptionDone='uploaded',
261                         workdir='install')
262
263 # generic builder
264
265
266 def generic_builder(id, libdir='', branch='', rsync=False):
267     filename = 'uploaded/buildbot_upload_' + id + '.zip'
268     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
269     test_script = '../blender.git/build_files/buildbot/slave_test.py'
270     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
271     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
272     unpack_script = 'master_unpack.py'
273
274     f = BuildFactory()
275     if libdir != '':
276         f.addStep(lib_svn_step(libdir))
277         # Special trick to make sure we always have all the libs.
278         if libdir.startswith("darwin"):
279             f.addStep(lib_svn_step("darwin"))
280
281     for submodule in ('blender-translations',
282                       'blender-addons',
283                       'blender-addons-contrib',
284                       'blender-dev-tools'):
285         f.addStep(git_submodule_step(submodule))
286
287     f.addStep(git_step(branch))
288     f.addStep(git_submodules_update())
289
290     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
291     f.addStep(Test(command=['python', test_script, id]))
292     f.addStep(ShellCommand(name='package',
293                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
294                            description='packaging',
295                            descriptionDone='packaged'))
296     if rsync:
297         f.addStep(rsync_step(id, branch, rsync_script))
298     else:
299         f.addStep(FileUpload(name='upload',
300                              slavesrc='buildbot_upload.zip',
301                              masterdest=filename,
302                              maxsize=150 * 1024 * 1024,
303                              workdir='install'))
304     f.addStep(MasterShellCommand(name='unpack',
305                                  command=['python2.7', unpack_script, filename],
306                                  description='unpacking',
307                                  descriptionDone='unpacked'))
308     return f
309
310 # Builders
311
312 add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
313 # add_builder(c, 'linux_glibc211_i686_cmake', '', generic_builder, hour=1)
314 # add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
315 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=3)
316 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=4)
317 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
318 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
319 add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=5)
320 add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=6)
321
322 # STATUS TARGETS
323 #
324 # 'status' is a list of Status Targets. The results of each build will be
325 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
326 # including web pages, email senders, and IRC bots.
327
328 c['status'] = []
329
330 from buildbot.status import html
331 from buildbot.status.web import authz
332 from buildbot.status.web import auth
333
334 users = []
335 for slave in master_private.slaves:
336     users += [(slave['name'], slave['password'])]
337
338 authz_cfg = authz.Authz(
339     auth=auth.BasicAuth(users),
340     # change any of these to True to enable; see the manual for more
341     # options
342     gracefulShutdown=False,
343     forceBuild=True,  # use this to test your slave once it is set up
344     forceAllBuilds=False,
345     pingBuilder=False,
346     stopBuild=True,
347     stopAllBuilds=False,
348     cancelPendingBuild=True,
349 )
350
351 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
352 #c['status'].append(html.WebStatus(http_port=8010))
353
354 # PROJECT IDENTITY
355
356 c['projectName'] = "Blender"
357 c['projectURL'] = "http://www.blender.org"
358
359 # the 'buildbotURL' string should point to the location where the buildbot's
360 # internal web server (usually the html.WebStatus page) is visible. This
361 # typically uses the port number set in the Waterfall 'status' entry, but
362 # with an externally-visible host name which the buildbot cannot figure out
363 # without some help.
364
365 c['buildbotURL'] = "http://builder.blender.org/"
366
367 # DB URL
368 #
369 # This specifies what database buildbot uses to store change and scheduler
370 # state.  You can leave this at its default for all but the largest
371 # installations.
372
373 c['db_url'] = "sqlite:///state.sqlite"