Buildbot: Cleanup unused code
[blender-staging.git] / build_files / buildbot / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "gooseberry", "experimental-build"]
11
12 """
13 Stock Twisted directory lister doesn't provide any information about last file
14 modification time, we hack the class a bit in order to have such functionaliity
15 :)
16 """
17
18 from buildbot.status.web.base import DirectoryLister
19
20
21 def get_files_and_directories(self, directory):
22     from twisted.web.static import (getTypeAndEncoding,
23                                     formatFileSize)
24     import urllib
25     import cgi
26     import time
27     import os
28     files = []
29     dirs = []
30     for path in directory:
31         url = urllib.quote(path, "/")
32         escapedPath = cgi.escape(path)
33         lastmodified = time.ctime(os.path.getmtime(
34             os.path.join(self.path, path)))
35         if os.path.isdir(os.path.join(self.path, path)):
36             url = url + '/'
37             dirs.append({'text': escapedPath + "/", 'href': url,
38                          'size': '', 'type': '[Directory]',
39                          'encoding': '',
40                          'lastmodified': lastmodified})
41         else:
42             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
43                                                     self.contentEncodings,
44                                                     self.defaultType)
45             try:
46                 size = os.stat(os.path.join(self.path, path)).st_size
47             except OSError:
48                 continue
49             files.append({
50                 'text': escapedPath, "href": url,
51                 'type': '[%s]' % mimetype,
52                 'encoding': (encoding and '[%s]' % encoding or ''),
53                 'size': formatFileSize(size),
54                 'lastmodified': lastmodified})
55     return dirs, files
56 DirectoryLister._getFilesAndDirectories = get_files_and_directories
57
58 # Dictionary that the buildmaster pays attention to.
59 c = BuildmasterConfig = {}
60
61 # BUILD SLAVES
62 #
63 # We load the slaves and their passwords from a separator file, so we can have
64 # this one in SVN.
65
66 from buildbot.buildslave import BuildSlave
67 import master_private
68
69 c['slaves'] = []
70
71 for slave in master_private.slaves:
72     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
73
74 # TCP port through which slaves connect
75
76 c['slavePortnum'] = 9989
77
78 # CHANGE SOURCES
79
80 from buildbot.changes.svnpoller import SVNPoller
81 from buildbot.changes.gitpoller import GitPoller
82
83 c['change_source'] = GitPoller(
84        'git://git.blender.org/blender.git',
85        pollinterval=1200)
86
87
88 # CODEBASES
89 #
90 # Allow to controll separately things like branches for each repo and submodules.
91
92 all_repositories = {
93     r'git://git.blender.org/blender.git': 'blender',
94     r'git://git.blender.org/blender-translations.git': 'blender-translations',
95     r'git://git.blender.org/blender-addons.git': 'blender-addons',
96     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
97     r'git://git.blender.org/scons.git': 'scons',
98     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
99 }
100
101
102 def codebaseGenerator(chdict):
103     return all_repositories[chdict['repository']]
104
105 c['codebaseGenerator'] = codebaseGenerator
106
107
108 # SCHEDULERS
109 #
110 # Decide how to react to incoming changes.
111
112 # from buildbot.scheduler import Scheduler
113 from buildbot.schedulers import timed, forcesched
114
115 c['schedulers'] = []
116
117
118 def schedule_force_build(name):
119     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
120         builderNames=[name],
121         codebases=[forcesched.CodebaseParameter(
122                 codebase="blender",
123                 branch=forcesched.ChoiceStringParameter(
124                     name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
125                 # Do not hide revision, can be handy!
126                 repository=forcesched.FixedParameter(name="repository", default="", hide=True),
127                 project=forcesched.FixedParameter(name="project", default="", hide=True)),
128             # For now, hide other codebases.
129             forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
130             forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
131             forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
132             forcesched.CodebaseParameter(hide=True, codebase="scons"),
133             forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
134         properties=[]))
135
136
137 def schedule_build(name, hour, minute=0):
138     for current_branch in NIGHT_SCHEDULE_BRANCHES:
139         scheduler_name = "nightly " + name
140         if current_branch:
141             scheduler_name += ' ' + current_branch
142         c['schedulers'].append(timed.Nightly(name=scheduler_name,
143             codebases={
144                 "blender": {"repository": ""},
145                 "blender-translations": {"repository": "", "branch": "master"},
146                 "blender-addons": {"repository": "", "branch": "master"},
147                 "blender-addons-contrib": {"repository": "", "branch": "master"},
148                 "scons": {"repository": "", "branch": "master"},
149                 "lib svn": {"repository": "", "branch": "trunk"}},
150             branch=current_branch,
151             builderNames=[name],
152             hour=hour,
153             minute=minute))
154
155
156 # BUILDERS
157 #
158 # The 'builders' list defines the Builders, which tell Buildbot how to
159 # perform a build: what steps, and which slaves can execute them.
160 # Note that any particular build will only take place on one slave.
161
162 from buildbot.process.factory import BuildFactory
163 from buildbot.process.properties import Interpolate
164 from buildbot.steps.source import SVN
165 from buildbot.steps.source import Git
166 from buildbot.steps.shell import ShellCommand
167 from buildbot.steps.shell import Compile
168 from buildbot.steps.shell import Test
169 from buildbot.steps.transfer import FileUpload
170 from buildbot.steps.master import MasterShellCommand
171 from buildbot.config import BuilderConfig
172
173 # add builder utility
174
175 c['builders'] = []
176 buildernames = []
177
178
179 def add_builder(c, name, libdir, factory, branch='',
180                 rsync=False, hour=3, minute=0):
181     slavenames = []
182
183     for slave in master_private.slaves:
184         if name in slave['builders']:
185             slavenames.append(slave['name'])
186
187     if len(slavenames) > 0:
188         f = factory(name, libdir, branch, rsync)
189         c['builders'].append(BuilderConfig(name=name,
190                                            slavenames=slavenames,
191                                            factory=f,
192                                            category='blender'))
193         buildernames.append(name)
194
195         schedule_build(name, hour, minute)
196         schedule_force_build(name)
197
198 # common steps
199
200
201 def git_submodule_step(submodule):
202     return Git(name=submodule + '.git',
203                repourl='git://git.blender.org/' + submodule + '.git',
204                mode='update',
205                codebase=submodule,
206                workdir=submodule + '.git')
207
208
209 def git_step(branch=''):
210     if branch:
211         return Git(name='blender.git',
212                    repourl='git://git.blender.org/blender.git',
213                    mode='update',
214                    branch=branch,
215                    codebase='blender',
216                    workdir='blender.git',
217                    submodules=True)
218     else:
219         return Git(name='blender.git',
220                    repourl='git://git.blender.org/blender.git',
221                    mode='update',
222                    codebase='blender',
223                    workdir='blender.git',
224                    submodules=True)
225
226
227 def git_submodules_update():
228     command = ['git', 'submodule', 'foreach', '--recursive',
229                'git', 'pull', 'origin', 'master']
230     return ShellCommand(name='Submodules Update',
231                         command=command,
232                         description='updating',
233                         descriptionDone='up to date',
234                         workdir='blender.git')
235
236
237 def lib_svn_step(dir):
238     return SVN(name='lib svn',
239                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
240                codebase='lib svn',
241                mode='update',
242                defaultBranch='trunk',
243                workdir='lib/' + dir)
244
245
246 def rsync_step(id, branch, rsync_script):
247     return ShellCommand(name='rsync',
248                         command=['python', rsync_script, id, branch],
249                         description='uploading',
250                         descriptionDone='uploaded',
251                         workdir='install')
252
253 # generic builder
254
255
256 def generic_builder(id, libdir='', branch='', rsync=False):
257     filename = 'uploaded/buildbot_upload_' + id + '.zip'
258     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
259     test_script = '../blender.git/build_files/buildbot/slave_test.py'
260     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
261     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
262     unpack_script = 'master_unpack.py'
263
264     f = BuildFactory()
265     if libdir != '':
266         f.addStep(lib_svn_step(libdir))
267
268     for submodule in ('blender-translations',
269                       'blender-addons',
270                       'blender-addons-contrib',
271                       'scons'):
272         f.addStep(git_submodule_step(submodule))
273
274     f.addStep(git_step(branch))
275     f.addStep(git_submodules_update())
276
277     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
278     f.addStep(Test(command=['python', test_script, id]))
279     f.addStep(ShellCommand(name='package',
280                            command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
281                            description='packaging',
282                            descriptionDone='packaged'))
283     if rsync:
284         f.addStep(rsync_step(id, branch, rsync_script))
285     else:
286         f.addStep(FileUpload(name='upload',
287                              slavesrc='buildbot_upload.zip',
288                              masterdest=filename,
289                              maxsize=150 * 1024 * 1024,
290                              workdir='install'))
291     f.addStep(MasterShellCommand(name='unpack',
292                                  command=['python', unpack_script, filename],
293                                  description='unpacking',
294                                  descriptionDone='unpacked'))
295     return f
296
297 # Builders
298
299 add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
300 add_builder(c, 'mac_i386_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=11)
301 add_builder(c, 'linux_glibc211_i386_cmake', '', generic_builder, hour=1)
302 add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
303 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
304 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
305
306 # STATUS TARGETS
307 #
308 # 'status' is a list of Status Targets. The results of each build will be
309 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
310 # including web pages, email senders, and IRC bots.
311
312 c['status'] = []
313
314 from buildbot.status import html
315 from buildbot.status.web import authz
316 from buildbot.status.web import auth
317
318 users = []
319 for slave in master_private.slaves:
320     users += [(slave['name'], slave['password'])]
321
322 authz_cfg = authz.Authz(
323     auth=auth.BasicAuth(users),
324     # change any of these to True to enable; see the manual for more
325     # options
326     gracefulShutdown=False,
327     forceBuild=True,  # use this to test your slave once it is set up
328     forceAllBuilds=False,
329     pingBuilder=False,
330     stopBuild=True,
331     stopAllBuilds=False,
332     cancelPendingBuild=True,
333 )
334
335 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
336 #c['status'].append(html.WebStatus(http_port=8010))
337
338 # PROJECT IDENTITY
339
340 c['projectName'] = "Blender"
341 c['projectURL'] = "http://www.blender.org"
342
343 # the 'buildbotURL' string should point to the location where the buildbot's
344 # internal web server (usually the html.WebStatus page) is visible. This
345 # typically uses the port number set in the Waterfall 'status' entry, but
346 # with an externally-visible host name which the buildbot cannot figure out
347 # without some help.
348
349 c['buildbotURL'] = "http://builder.blender.org/"
350
351 # DB URL
352 #
353 # This specifies what database buildbot uses to store change and scheduler
354 # state.  You can leave this at its default for all but the largest
355 # installations.
356
357 c['db_url'] = "sqlite:///state.sqlite"