Cleanup: pep8 & redundant vars
[blender.git] / build_files / buildbot / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 """
7 Stock Twisted directory lister doesn't provide any information about last file
8 modification time, we hack the class a bit in order to have such functionaliity
9 :)
10 """
11
12 from buildbot.status.web.base import DirectoryLister
13
14
15 def get_files_and_directories(self, directory):
16     from twisted.web.static import (getTypeAndEncoding,
17                                     formatFileSize)
18     import urllib
19     import cgi
20     import time
21     import os
22     files = []
23     dirs = []
24     for path in directory:
25         url = urllib.quote(path, "/")
26         escapedPath = cgi.escape(path)
27         lastmodified = time.ctime(os.path.getmtime(
28             os.path.join(self.path, path)))
29         if os.path.isdir(os.path.join(self.path, path)):
30             url = url + '/'
31             dirs.append({'text': escapedPath + "/", 'href': url,
32                          'size': '', 'type': '[Directory]',
33                          'encoding': '',
34                          'lastmodified': lastmodified})
35         else:
36             mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
37                                                     self.contentEncodings,
38                                                     self.defaultType)
39             try:
40                 size = os.stat(os.path.join(self.path, path)).st_size
41             except OSError:
42                 continue
43             files.append({
44                 'text': escapedPath, "href": url,
45                 'type': '[%s]' % mimetype,
46                 'encoding': (encoding and '[%s]' % encoding or ''),
47                 'size': formatFileSize(size),
48                 'lastmodified': lastmodified})
49     return dirs, files
50 DirectoryLister._getFilesAndDirectories = get_files_and_directories
51
52 # Dictionary that the buildmaster pays attention to.
53 c = BuildmasterConfig = {}
54
55 # BUILD SLAVES
56 #
57 # We load the slaves and their passwords from a separator file, so we can have
58 # this one in SVN.
59
60 from buildbot.buildslave import BuildSlave
61 import master_private
62
63 c['slaves'] = []
64
65 for slave in master_private.slaves:
66     c['slaves'].append(BuildSlave(slave['name'], slave['password']))
67
68 # TCP port through which slaves connect
69
70 c['slavePortnum'] = 9989
71
72 # CHANGE SOURCES
73
74 from buildbot.changes.svnpoller import SVNPoller
75 from buildbot.changes.gitpoller import GitPoller
76
77 c['change_source'] = GitPoller(
78        'git://git.blender.org/blender.git',
79        pollinterval=1200)
80
81 # SCHEDULERS
82 #
83 # Decide how to react to incoming changes.
84
85 # from buildbot.scheduler import Scheduler
86 from buildbot.schedulers import timed, forcesched
87
88 c['schedulers'] = []
89
90
91 def schedule_force_build(name):
92     c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
93         builderNames=[name],
94         branch=forcesched.FixedParameter(name="branch", default=""),
95         revision=forcesched.FixedParameter(name="revision", default=""),
96         repository=forcesched.FixedParameter(name="repository", default=""),
97         project=forcesched.FixedParameter(name="project", default=""),
98         properties=[]))
99
100
101 def schedule_build(name, hour, minute=0):
102     c['schedulers'].append(timed.Nightly(name='nightly ' + name,
103         branch=None,  # default branch
104         builderNames=[name],
105         hour=hour,
106         minute=minute))
107
108
109 # BUILDERS
110 #
111 # The 'builders' list defines the Builders, which tell Buildbot how to
112 # perform a build: what steps, and which slaves can execute them.
113 # Note that any particular build will only take place on one slave.
114
115 from buildbot.process.factory import BuildFactory
116 from buildbot.steps.source import SVN
117 from buildbot.steps.source import Git
118 from buildbot.steps.shell import ShellCommand
119 from buildbot.steps.shell import Compile
120 from buildbot.steps.shell import Test
121 from buildbot.steps.transfer import FileUpload
122 from buildbot.steps.master import MasterShellCommand
123 from buildbot.config import BuilderConfig
124
125 # add builder utility
126
127 c['builders'] = []
128 buildernames = []
129
130
131 def add_builder(c, name, libdir, factory, branch='',
132                 rsync=False, hour=3, minute=0):
133     slavenames = []
134
135     for slave in master_private.slaves:
136         if name in slave['builders']:
137             slavenames.append(slave['name'])
138
139     if len(slavenames) > 0:
140         f = factory(name, libdir, branch, rsync)
141         c['builders'].append(BuilderConfig(name=name,
142                                            slavenames=slavenames,
143                                            factory=f,
144                                            category='blender'))
145         buildernames.append(name)
146
147         schedule_build(name, hour, minute)
148         schedule_force_build(name)
149
150 # common steps
151
152
153 def git_submodule_step(submodule):
154     return Git(name=submodule + '.git',
155                repourl='git://git.blender.org/' + submodule + '.git',
156                mode='update',
157                workdir=submodule + '.git')
158
159
160 def git_step(branch=''):
161     if branch:
162         return Git(name='blender.git',
163                    repourl='git://git.blender.org/blender.git',
164                    mode='update',
165                    branch=branch,
166                    workdir='blender.git',
167                    submodules=True)
168     else:
169         return Git(name='blender.git',
170                    repourl='git://git.blender.org/blender.git',
171                    mode='update',
172                    workdir='blender.git',
173                    submodules=True)
174
175
176 def git_submodules_update():
177     command = ['git', 'submodule', 'foreach', '--recursive',
178                'git', 'pull', 'origin', 'master']
179     return ShellCommand(name='Submodules Update',
180                         command=command,
181                         description='updating',
182                         descriptionDone='up to date',
183                         workdir='blender.git')
184
185
186 def lib_svn_step(dir):
187     return SVN(name='lib svn',
188                baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
189                mode='update',
190                defaultBranch='trunk',
191                workdir='lib/' + dir)
192
193
194 def rsync_step(id, branch, rsync_script):
195     return ShellCommand(name='rsync',
196                         command=['python', rsync_script, id, branch],
197                         description='uploading',
198                         descriptionDone='uploaded',
199                         workdir='install')
200
201 # generic builder
202
203
204 def generic_builder(id, libdir='', branch='', rsync=False):
205     filename = 'uploaded/buildbot_upload_' + id + '.zip'
206     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
207     test_script = '../blender.git/build_files/buildbot/slave_test.py'
208     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
209     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
210     unpack_script = 'master_unpack.py'
211
212     f = BuildFactory()
213     if libdir != '':
214         f.addStep(lib_svn_step(libdir))
215
216     for submodule in ('blender-translations',
217                       'blender-addons',
218                       'blender-addons-contrib',
219                       'scons'):
220         f.addStep(git_submodule_step(submodule))
221
222     f.addStep(git_step(branch))
223     f.addStep(git_submodules_update())
224
225     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
226     f.addStep(Test(command=['python', test_script, id]))
227     f.addStep(ShellCommand(name='package',
228                            command=['python', pack_script, id, branch],
229                            description='packaging',
230                            descriptionDone='packaged'))
231     if rsync:
232         f.addStep(rsync_step(id, branch, rsync_script))
233     elif id.find('cmake') != -1:
234         f.addStep(FileUpload(name='upload',
235                              slavesrc='buildbot_upload.zip',
236                              masterdest=filename,
237                              maxsize=150 * 1024 * 1024))
238     else:
239         f.addStep(FileUpload(name='upload',
240                              slavesrc='buildbot_upload.zip',
241                              masterdest=filename,
242                              maxsize=150 * 1024 * 1024,
243                              workdir='install'))
244     f.addStep(MasterShellCommand(name='unpack',
245                                  command=['python', unpack_script, filename],
246                                  description='unpacking',
247                                  descriptionDone='unpacked'))
248     return f
249
250 # builders
251
252 add_builder(c, 'mac_x86_64_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=5)
253 add_builder(c, 'mac_i386_10_6_scons', 'darwin-9.x.universal', generic_builder, hour=11)
254 add_builder(c, 'linux_glibc211_i386_scons', '', generic_builder, hour=1)
255 add_builder(c, 'linux_glibc211_x86_64_scons', '', generic_builder, hour=2)
256 add_builder(c, 'win32_scons', 'windows', generic_builder, hour=1)
257 add_builder(c, 'win64_scons', 'win64', generic_builder, hour=2)
258 #add_builder(c, 'win32_scons_vc2012', 'windows_vc11', generic_builder, hour=1)
259 #add_builder(c, 'win64_scons_vc2012', 'win64_vc11', generic_builder, hour=2)
260 add_builder(c, 'win32_scons_vc2013', 'windows_vc12', generic_builder, hour=1)
261 add_builder(c, 'win64_scons_vc2013', 'win64_vc12', generic_builder, hour=2)
262 #add_builder(c, 'mingw_win32_scons', 'mingw32', generic_builder, hour=4)
263 add_builder(c, 'mingw_win64_scons', 'mingw64', generic_builder, hour=3)
264 #add_builder(c, 'freebsd_i386_cmake', '', generic_builder, hour=1)
265 #add_builder(c, 'freebsd_x86_64_cmake', '', generic_builder, hour=2)
266
267 # Multiview branch
268 add_builder(c, 'multiview_win64_scons', 'win64', generic_builder, 'multiview', hour=4)
269 add_builder(c, 'multiview_win32_scons', 'windows', generic_builder, 'multiview', hour=5)
270
271 # STATUS TARGETS
272 #
273 # 'status' is a list of Status Targets. The results of each build will be
274 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
275 # including web pages, email senders, and IRC bots.
276
277 c['status'] = []
278
279 from buildbot.status import html
280 from buildbot.status.web import authz
281 from buildbot.status.web import auth
282
283 users = []
284 for slave in master_private.slaves:
285     users += [(slave['name'], slave['password'])]
286
287 authz_cfg = authz.Authz(
288     auth=auth.BasicAuth(users),
289     # change any of these to True to enable; see the manual for more
290     # options
291     gracefulShutdown=False,
292     forceBuild=True,  # use this to test your slave once it is set up
293     forceAllBuilds=False,
294     pingBuilder=False,
295     stopBuild=True,
296     stopAllBuilds=False,
297     cancelPendingBuild=True,
298 )
299
300 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
301 #c['status'].append(html.WebStatus(http_port=8010))
302
303 # PROJECT IDENTITY
304
305 c['projectName'] = "Blender"
306 c['projectURL'] = "http://www.blender.org"
307
308 # the 'buildbotURL' string should point to the location where the buildbot's
309 # internal web server (usually the html.WebStatus page) is visible. This
310 # typically uses the port number set in the Waterfall 'status' entry, but
311 # with an externally-visible host name which the buildbot cannot figure out
312 # without some help.
313
314 c['buildbotURL'] = "http://builder.blender.org/"
315
316 # DB URL
317 #
318 # This specifies what database buildbot uses to store change and scheduler
319 # state.  You can leave this at its default for all but the largest
320 # installations.
321
322 c['db_url'] = "sqlite:///state.sqlite"