Synchronize changes with server
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "gooseberry", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to control separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
98 }
99
100
101 def codebaseGenerator(chdict):
102     return all_repositories[chdict['repository']]
103
104 c['codebaseGenerator'] = codebaseGenerator
105
106
107 # SCHEDULERS
108 #
109 # Decide how to react to incoming changes.
110
111 # from buildbot.scheduler import Scheduler
112 from buildbot.schedulers import timed, forcesched
113
114 c['schedulers'] = []
115
116
117 def schedule_force_build(name):
118     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
119         builderNames=[name],
120         codebases=[forcesched.CodebaseParameter(
121                 codebase="blender",
122                 branch=forcesched.ChoiceStringParameter(
123                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
124                 # Do not hide revision, can be handy!
125                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
126                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
127             # For now, hide other codebases.
128             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
129             forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
130             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
131             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
132         properties=[]))
133
134
135 def schedule_build(name, hour, minute=0):
136     for current_branch in NIGHT_SCHEDULE_BRANCHES:
137         scheduler_name = "nightly " + name
138         if current_branch:
139             scheduler_name += ' ' + current_branch
140         c['schedulers'].append(timed.Nightly(name=scheduler_name,
141             codebases={
142                 "blender": {"repository": ""},
143                 "blender-translations": {"repository": "", "branch": "master"},
144                 "blender-addons": {"repository": "", "branch": "master"},
145                 "blender-addons-contrib": {"repository": "", "branch": "master"},
146                 "lib svn": {"repository": "", "branch": "trunk"}},
147             branch=current_branch,
148             builderNames=[name],
149             hour=hour,
150             minute=minute))
151
152
153 # BUILDERS
154 #
155 # The 'builders' list defines the Builders, which tell Buildbot how to
156 # perform a build: what steps, and which slaves can execute them.
157 # Note that any particular build will only take place on one slave.
158
159 from buildbot.process.factory import BuildFactory
160 from buildbot.process.properties import Interpolate
161 from buildbot.steps.source import SVN
162 from buildbot.steps.source import Git
163 from buildbot.steps.shell import ShellCommand
164 from buildbot.steps.shell import Compile
165 from buildbot.steps.shell import Test
166 from buildbot.steps.transfer import FileUpload
167 from buildbot.steps.master import MasterShellCommand
168 from buildbot.config import BuilderConfig
169
170 # add builder utility
171
172 c['builders'] = []
173 buildernames = []
174
175
176 def add_builder(c, name, libdir, factory, branch='',
177                 rsync=False, hour=3, minute=0):
178     slavenames = []
179
180     for slave in master_private.slaves:
181         if name in slave['builders']:
182             slavenames.append(slave['name'])
183
184     if len(slavenames) > 0:
185         f = factory(name, libdir, branch, rsync)
186         c['builders'].append(BuilderConfig(name=name,
187                                            slavenames=slavenames,
188                                            factory=f,
189                                            category='blender'))
190         buildernames.append(name)
191
192         schedule_build(name, hour, minute)
193         schedule_force_build(name)
194
195 # common steps
196
197
198 def git_submodule_step(submodule):
199     return Git(name=submodule + '.git',
200                repourl='git://git.blender.org/' + submodule + '.git',
201                mode='update',
202                codebase=submodule,
203                workdir=submodule + '.git')
204
205
206 def git_step(branch=''):
207     if branch:
208         return Git(name='blender.git',
209                    repourl='git://git.blender.org/blender.git',
210                    mode='update',
211                    branch=branch,
212                    codebase='blender',
213                    workdir='blender.git',
214                    submodules=True)
215     else:
216         return Git(name='blender.git',
217                    repourl='git://git.blender.org/blender.git',
218                    mode='update',
219                    codebase='blender',
220                    workdir='blender.git',
221                    submodules=True)
222
223
224 def git_submodules_update():
225     command = ['git', 'submodule', 'foreach', '--recursive',
226                'git', 'pull', 'origin', 'master']
227     return ShellCommand(name='Submodules Update',
228                         command=command,
229                         description='updating',
230                         descriptionDone='up to date',
231                         workdir='blender.git')
232
233
234 def lib_svn_step(dir):
235     return SVN(name='lib svn',
236                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
237                codebase='lib svn',
238                mode='update',
239                defaultBranch='trunk',
240                workdir='lib/' + dir)
241
242
243 def rsync_step(id, branch, rsync_script):
244     return ShellCommand(name='rsync',
245                         command=['python', rsync_script, id, branch],
246                         description='uploading',
247                         descriptionDone='uploaded',
248                         workdir='install')
249
250 # generic builder
251
252
253 def generic_builder(id, libdir='', branch='', rsync=False):
254     filename = 'uploaded/buildbot_upload_' + id + '.zip'
255     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
256     test_script = '../blender.git/build_files/buildbot/slave_test.py'
257     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
258     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
259     unpack_script = 'master_unpack.py'
260
261     f = BuildFactory()
262     if libdir != '':
263         f.addStep(lib_svn_step(libdir))
264
265     for submodule in ('blender-translations',
266                       'blender-addons',
267                       'blender-addons-contrib'):
268         f.addStep(git_submodule_step(submodule))
269
270     f.addStep(git_step(branch))
271     f.addStep(git_submodules_update())
272
273     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
274     f.addStep(Test(command=['python', test_script, id]))
275     f.addStep(ShellCommand(name='package',
276                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
277                            description='packaging',
278                            descriptionDone='packaged'))
279     if rsync:
280         f.addStep(rsync_step(id, branch, rsync_script))
281     else:
282         f.addStep(FileUpload(name='upload',
283                              slavesrc='buildbot_upload.zip',
284                              masterdest=filename,
285                              maxsize=150 * 1024 * 1024,
286                              workdir='install'))
287     f.addStep(MasterShellCommand(name='unpack',
288                                  command=['python2.7', unpack_script, filename],
289                                  description='unpacking',
290                                  descriptionDone='unpacked'))
291     return f
292
293 # Builders
294
295 add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
296 add_builder(c, 'linux_glibc211_i686_cmake', '', generic_builder, hour=1)
297 add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
298 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=3)
299 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=4)
300 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
301 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
302 add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=5)
303
304 # STATUS TARGETS
305 #
306 # 'status' is a list of Status Targets. The results of each build will be
307 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
308 # including web pages, email senders, and IRC bots.
309
310 c['status'] = []
311
312 from buildbot.status import html
313 from buildbot.status.web import authz
314 from buildbot.status.web import auth
315
316 users = []
317 for slave in master_private.slaves:
318     users += [(slave['name'], slave['password'])]
319
320 authz_cfg = authz.Authz(
321     auth=auth.BasicAuth(users),
322     # change any of these to True to enable; see the manual for more
323     # options
324     gracefulShutdown=False,
325     forceBuild=True,  # use this to test your slave once it is set up
326     forceAllBuilds=False,
327     pingBuilder=False,
328     stopBuild=True,
329     stopAllBuilds=False,
330     cancelPendingBuild=True,
331 )
332
333 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
334 #c['status'].append(html.WebStatus(http_port=8010))
335
336 # PROJECT IDENTITY
337
338 c['projectName'] = "Blender"
339 c['projectURL'] = "http://www.blender.org"
340
341 # the 'buildbotURL' string should point to the location where the buildbot's
342 # internal web server (usually the html.WebStatus page) is visible. This
343 # typically uses the port number set in the Waterfall 'status' entry, but
344 # with an externally-visible host name which the buildbot cannot figure out
345 # without some help.
346
347 c['buildbotURL'] = "http://builder.blender.org/"
348
349 # DB URL
350 #
351 # This specifies what database buildbot uses to store change and scheduler
352 # state.  You can leave this at its default for all but the largest
353 # installations.
354
355 c['db_url'] = "sqlite:///state.sqlite"