Update for moving blender2.8 to master, master to blender2.7.
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.7"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.7", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to control separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
99 }
100
101
102 def codebaseGenerator(chdict):
103     return all_repositories[chdict['repository']]
104
105 c['codebaseGenerator'] = codebaseGenerator
106
107
108 # SCHEDULERS
109 #
110 # Decide how to react to incoming changes.
111
112 # from buildbot.scheduler import Scheduler
113 from buildbot.schedulers import timed, forcesched
114
115 c['schedulers'] = []
116
117
118 def schedule_force_build(name):
119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
120         builderNames=[name],
121         codebases=[forcesched.CodebaseParameter(
122                 codebase="blender",
123                 branch=forcesched.ChoiceStringParameter(
124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
125                 # Do not hide revision, can be handy!
126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
128             # For now, hide other codebases.
129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
130             forcesched.CodebaseParameter(
131                 codebase="blender-addons",
132                 branch=forcesched.ChoiceStringParameter(
133                     name="branch", choices=["master", "blender2.7"], default="master"),
134                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
135                 project=forcesched.FixedParameter(name="project", default="", hide=True),
136                 revision=forcesched.FixedParameter(name="revision", default="", hide=True),
137             ),
138             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
139             forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
140             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
141         properties=[]))
142
143
144 def schedule_build(name, hour, minute=0):
145     for current_branch in NIGHT_SCHEDULE_BRANCHES:
146         scheduler_name = "nightly " + name
147         if current_branch:
148             scheduler_name += ' ' + current_branch
149         # Use special addons submodule branch when building blender2.7 branch.
150         addons_branch = "master"
151         if current_branch == "blender2.7":
152             addons_branch = "blender2.7"
153         c['schedulers'].append(timed.Nightly(name=scheduler_name,
154             codebases={
155                 "blender": {"repository": ""},
156                 "blender-translations": {"repository": "", "branch": "master"},
157                 "blender-addons": {"repository": "", "branch": addons_branch},
158                 "blender-addons-contrib": {"repository": "", "branch": "master"},
159                 "blender-dev-tools": {"repository": "", "branch": "master"},
160                 "lib svn": {"repository": "", "branch": "trunk"}},
161             branch=current_branch,
162             builderNames=[name],
163             hour=hour,
164             minute=minute))
165
166
167 # BUILDERS
168 #
169 # The 'builders' list defines the Builders, which tell Buildbot how to
170 # perform a build: what steps, and which slaves can execute them.
171 # Note that any particular build will only take place on one slave.
172
173 from buildbot.process.factory import BuildFactory
174 from buildbot.process.properties import Interpolate
175 from buildbot.steps.source import SVN
176 from buildbot.steps.source import Git
177 from buildbot.steps.shell import ShellCommand
178 from buildbot.steps.shell import Compile
179 from buildbot.steps.shell import Test
180 from buildbot.steps.transfer import FileUpload
181 from buildbot.steps.master import MasterShellCommand
182 from buildbot.config import BuilderConfig
183
184 # add builder utility
185
186 c['builders'] = []
187 buildernames = []
188
189
190 def add_builder(c, name, libdir, factory, branch='',
191                 rsync=False, hour=3, minute=0):
192     slavenames = []
193
194     for slave in master_private.slaves:
195         if name in slave['builders']:
196             slavenames.append(slave['name'])
197
198     if len(slavenames) > 0:
199         f = factory(name, libdir, branch, rsync)
200         c['builders'].append(BuilderConfig(name=name,
201                                            slavenames=slavenames,
202                                            factory=f,
203                                            category='blender'))
204         buildernames.append(name)
205
206         schedule_build(name, hour, minute)
207         schedule_force_build(name)
208
209 # common steps
210
211
212 def git_submodule_step(submodule):
213     return Git(name=submodule + '.git',
214                repourl='git://git.blender.org/' + submodule + '.git',
215                mode='update',
216                codebase=submodule,
217                workdir=submodule + '.git')
218
219
220 def git_step(branch=''):
221     if branch:
222         return Git(name='blender.git',
223                    repourl='git://git.blender.org/blender.git',
224                    mode='update',
225                    branch=branch,
226                    codebase='blender',
227                    workdir='blender.git',
228                    submodules=True)
229     else:
230         return Git(name='blender.git',
231                    repourl='git://git.blender.org/blender.git',
232                    mode='update',
233                    codebase='blender',
234                    workdir='blender.git',
235                    submodules=True)
236
237
238 def git_submodules_update():
239     command = ['git', 'submodule', 'update', '--remote']
240     return ShellCommand(name='Submodules Update',
241                         command=command,
242                         description='updating',
243                         descriptionDone='up to date',
244                         workdir='blender.git')
245
246
247 def lib_svn_step(dir):
248     return SVN(name='lib svn',
249                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
250                codebase='lib svn',
251                mode='update',
252                defaultBranch='trunk',
253                workdir='lib/' + dir)
254
255
256 def rsync_step(id, branch, rsync_script):
257     return ShellCommand(name='rsync',
258                         command=['python', rsync_script, id, branch],
259                         description='uploading',
260                         descriptionDone='uploaded',
261                         workdir='install')
262
263 # generic builder
264
265
266 def generic_builder(id, libdir='', branch='', rsync=False):
267     filename = 'uploaded/buildbot_upload_' + id + '.zip'
268     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
269     test_script = '../blender.git/build_files/buildbot/slave_test.py'
270     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
271     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
272     unpack_script = 'master_unpack.py'
273
274     f = BuildFactory()
275     if libdir != '':
276         f.addStep(lib_svn_step(libdir))
277
278     for submodule in ('blender-translations',
279                       'blender-addons',
280                       'blender-addons-contrib',
281                       'blender-dev-tools'):
282         f.addStep(git_submodule_step(submodule))
283
284     f.addStep(git_step(branch))
285     f.addStep(git_submodules_update())
286
287     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
288     f.addStep(Test(command=['python', test_script, id]))
289     f.addStep(ShellCommand(name='package',
290                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
291                            description='packaging',
292                            descriptionDone='packaged'))
293     if rsync:
294         f.addStep(rsync_step(id, branch, rsync_script))
295     else:
296         f.addStep(FileUpload(name='upload',
297                              slavesrc='buildbot_upload.zip',
298                              masterdest=filename,
299                              maxsize=180 * 1024 * 1024,
300                              workdir='install'))
301     f.addStep(MasterShellCommand(name='unpack',
302                                  command=['python2.7', unpack_script, filename],
303                                  description='unpacking',
304                                  descriptionDone='unpacked'))
305     return f
306
307 # Builders
308
309 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
310 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2)
311 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1)
312 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1)
313 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2)
314 add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3)
315 add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4)
316
317 # STATUS TARGETS
318 #
319 # 'status' is a list of Status Targets. The results of each build will be
320 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
321 # including web pages, email senders, and IRC bots.
322
323 c['status'] = []
324
325 from buildbot.status import html
326 from buildbot.status.web import authz
327 from buildbot.status.web import auth
328
329 users = []
330 for slave in master_private.slaves:
331     users += [(slave['name'], slave['password'])]
332
333 authz_cfg = authz.Authz(
334     auth=auth.BasicAuth(users),
335     # change any of these to True to enable; see the manual for more
336     # options
337     gracefulShutdown=False,
338     forceBuild=True,  # use this to test your slave once it is set up
339     forceAllBuilds=False,
340     pingBuilder=False,
341     stopBuild=True,
342     stopAllBuilds=False,
343     cancelPendingBuild=True,
344 )
345
346 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
347 #c['status'].append(html.WebStatus(http_port=8010))
348
349 # PROJECT IDENTITY
350
351 c['projectName'] = "Blender"
352 c['projectURL'] = "http://www.blender.org"
353
354 # the 'buildbotURL' string should point to the location where the buildbot's
355 # internal web server (usually the html.WebStatus page) is visible. This
356 # typically uses the port number set in the Waterfall 'status' entry, but
357 # with an externally-visible host name which the buildbot cannot figure out
358 # without some help.
359
360 c['buildbotURL'] = "http://builder.blender.org/"
361
362 # DB URL
363 #
364 # This specifies what database buildbot uses to store change and scheduler
365 # state.  You can leave this at its default for all but the largest
366 # installations.
367
368 c['db_url'] = "sqlite:///state.sqlite"