Remove 32 bit buildbots
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = ["master"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "experimental-build"]
11
12 # List of branches availble for addons branch selection.
13 ADDONS_BRANCHES = ["master"]
14
15 # List of branches availble for libraries branch selection.
16 LIB_BRANCHES = ["trunk"]
17
18 # Dictionary that the buildmaster pays attention to.
19 c = BuildmasterConfig = {}
20
21 # BUILD WORKERS
22 #
23 # We load the slaves and their passwords from a separator file, so we can have
24 # this one in SVN.
25
26 from buildbot.worker import Worker
27 import master_private
28
29 c['workers'] = []
30 for slave in master_private.slaves:
31     c['workers'].append(Worker(slave['name'], slave['password']))
32
33 # TCP port through which slaves connect
34 c['protocols'] = {
35     "pb": {
36         "port": "tcp:{}".format(9989)
37     }
38 }
39 # CHANGE SOURCES
40
41 from buildbot.changes.svnpoller import SVNPoller
42 from buildbot.changes.gitpoller import GitPoller
43
44 c['change_source'] = GitPoller('git://git.blender.org/blender.git',
45                                pollinterval=1200)
46
47 # CODEBASES
48 #
49 # Allow to control separately things like branches for each repo and submodules.
50
51 all_repositories = {
52     r'git://git.blender.org/blender.git': 'blender',
53     r'git://git.blender.org/blender-translations.git': 'blender-translations',
54     r'git://git.blender.org/blender-addons.git': 'blender-addons',
55     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
56     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
57     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
58 }
59
60
61 def codebaseGenerator(chdict):
62     return all_repositories[chdict['repository']]
63
64
65 c['codebaseGenerator'] = codebaseGenerator
66
67
68 # SCHEDULERS
69 #
70 # Decide how to react to incoming changes.
71
72 from buildbot.schedulers import timed, forcesched
73
74 c['schedulers'] = []
75
76
77 def schedule_force_build(name):
78     """
79     Makes it possible to have "Force Build" for the given builder.
80     Makes sure only reasonabel subset of properties are exposed.
81     """
82     c['schedulers'].append(forcesched.ForceScheduler(
83         name='force_' + name,
84         buttonName="Force Build",
85         builderNames=[name],
86         codebases=[forcesched.CodebaseParameter(
87                 codebase="blender",
88                 branch=forcesched.ChoiceStringParameter(
89                     name="branch",
90                     choices=FORCE_SCHEDULE_BRANCHES,
91                     default="master"),
92                 # Do not hide revision, can be handy!
93                 repository=forcesched.FixedParameter(
94                     name="repository",
95                     default="",
96                     hide=True),
97                 project=forcesched.FixedParameter(
98                     name="project",
99                     default="",
100                     hide=True)),
101                    # For now, hide other codebases.
102                    forcesched.CodebaseParameter(
103                        hide=True,
104                        codebase="blender-translations"),
105                    forcesched.CodebaseParameter(
106                        codebase="blender-addons",
107                        branch=forcesched.ChoiceStringParameter(
108                            name="branch",
109                            choices=ADDONS_BRANCHES,
110                            default="master"),
111                        repository=forcesched.FixedParameter(name="repository",
112                                                             default="",
113                                                             hide=True),
114                        project=forcesched.FixedParameter(name="project",
115                                                          default="",
116                                                          hide=True),
117                        revision=forcesched.FixedParameter(name="revision",
118                                                           default="",
119                                                           hide=True),
120                    ),
121                    forcesched.CodebaseParameter(
122                        codebase="blender-addons-contrib",
123                        branch=forcesched.ChoiceStringParameter(
124                            name="branch",
125                            choices=ADDONS_BRANCHES,
126                            default="master"),
127                        repository=forcesched.FixedParameter(name="repository",
128                                                             default="",
129                                                             hide=True),
130                        project=forcesched.FixedParameter(name="project",
131                                                          default="",
132                                                          hide=True),
133                        revision=forcesched.FixedParameter(name="revision",
134                                                           default="",
135                                                           hide=True),
136                    ),
137                    forcesched.CodebaseParameter(
138                        hide=True,
139                        codebase="blender-dev-tools"),
140                    forcesched.CodebaseParameter(
141                        codebase="lib svn",
142                        branch=forcesched.ChoiceStringParameter(
143                            name="branch",
144                            choices=LIB_BRANCHES,
145                            default="trunk"),
146                        repository=forcesched.FixedParameter(name="repository",
147                                                             default="",
148                                                             hide=True),
149                        project=forcesched.FixedParameter(name="project",
150                                                          default="",
151                                                          hide=True),
152                        revision=forcesched.FixedParameter(name="revision",
153                                                           default="",
154                                                           hide=True),
155                    ),
156         ],
157         properties=[]))
158
159
160 def schedule_nightly_build(name, hour, minute=0):
161     """
162     Creates scheduler for nightly builds for a given builder.
163     """
164     for current_branch in NIGHT_SCHEDULE_BRANCHES:
165         scheduler_name = "nightly_" + name
166         if current_branch:
167             scheduler_name += ' ' + current_branch
168         addons_branch = "master"
169         c['schedulers'].append(timed.Nightly(
170             name=scheduler_name,
171             codebases={
172                 "blender": {"repository": "",
173                             "branch": current_branch},
174                 "blender-translations": {"repository": "",
175                                          "branch": "master"},
176                 "blender-addons": {"repository": "",
177                                    "branch": addons_branch},
178                 "blender-addons-contrib": {"repository": "",
179                                            "branch": addons_branch},
180                 "blender-dev-tools": {"repository": "",
181                                       "branch": "master"},
182                 "lib svn": {"repository": "",
183                             "branch": "trunk"}},
184             branch=current_branch,
185             builderNames=[name],
186             hour=hour,
187             minute=minute))
188
189
190 # BUILDERS
191 #
192 # The 'builders' list defines the Builders, which tell Buildbot how to
193 # perform a build: what steps, and which slaves can execute them.
194 # Note that any particular build will only take place on one slave.
195
196 from buildbot.config import BuilderConfig
197 from buildbot.plugins import steps, util
198 from buildbot.process.factory import BuildFactory
199 from buildbot.process.properties import Interpolate
200 from buildbot.steps.shell import ShellCommand
201 from buildbot.steps.shell import Compile
202 from buildbot.steps.shell import Test
203 from buildbot.steps.transfer import FileUpload
204 from buildbot.steps.master import MasterShellCommand
205
206 # add builder utility
207
208 c['builders'] = []
209 buildernames = []
210
211
212 def add_builder(c, name, libdir, factory, branch='',
213                 rsync=False, hour=3, minute=0):
214     workernames = []
215
216     for slave in master_private.slaves:
217         if name in slave['builders']:
218             workernames.append(slave['name'])
219
220     if workernames:
221         f = factory(name, libdir, branch, rsync)
222         c['builders'].append(BuilderConfig(name=name,
223                                            workernames=workernames,
224                                            factory=f,
225                                            tags=['blender']))
226         buildernames.append(name)
227
228         schedule_nightly_build(name, hour, minute)
229         schedule_force_build(name)
230
231
232 # common steps
233
234 def git_submodule_step(submodule):
235     return steps.Git(name=submodule + '.git',
236                      repourl='git://git.blender.org/' + submodule + '.git',
237                      mode='incremental',
238                      codebase=submodule,
239                      workdir=submodule + '.git')
240
241
242 def git_step(branch=''):
243     if branch:
244         return steps.Git(name='blender.git',
245                          repourl='git://git.blender.org/blender.git',
246                          mode='incremental',
247                          branch=branch,
248                          codebase='blender',
249                          workdir='blender.git',
250                          submodules=True)
251     else:
252         return steps.Git(name='blender.git',
253                          repourl='git://git.blender.org/blender.git',
254                          mode='incremental',
255                          codebase='blender',
256                          workdir='blender.git',
257                          submodules=True)
258
259
260 def git_submodules_update():
261     command = ['git', 'submodule', 'update', '--remote']
262     return ShellCommand(name='Submodules Update',
263                         command=command,
264                         description='updating',
265                         descriptionDone='up to date',
266                         workdir='blender.git')
267
268
269 def lib_svn_step(dir):
270     # TODO(sergey): For some reason interpolation is always giving empty branch.
271     # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
272     lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
273     return steps.SVN(name='lib svn ' + dir,
274                      repourl=util.Interpolate(lib_repo + dir),
275                      codebase='lib svn',
276                      mode='incremental',
277                      workdir='lib/' + dir)
278
279
280 def rsync_step(id, branch, rsync_script):
281     return ShellCommand(name='rsync',
282                         command=['python', rsync_script, id, branch],
283                         description='uploading',
284                         descriptionDone='uploaded',
285                         workdir='install')
286
287
288 # generic builder
289
290 def generic_builder(id, libdir='', branch='', rsync=False):
291     filename = 'uploaded/buildbot_upload_' + id + '.zip'
292     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
293     test_script = '../blender.git/build_files/buildbot/slave_test.py'
294     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
295     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
296     unpack_script = 'master_unpack.py'
297
298     f = BuildFactory()
299     if libdir != '':
300         f.addStep(lib_svn_step(libdir))
301
302     f.addStep(lib_svn_step('tests'))
303
304     for submodule in ('blender-translations',
305                       'blender-addons',
306                       'blender-addons-contrib',
307                       'blender-dev-tools'):
308         f.addStep(git_submodule_step(submodule))
309
310     f.addStep(git_step(branch))
311     f.addStep(git_submodules_update())
312
313     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
314     f.addStep(Test(command=['python', test_script, id]))
315     f.addStep(ShellCommand(
316         name='package',
317         command=['python',
318                  pack_script,
319                  id,
320                  branch or Interpolate('%(src:blender:branch)s')],
321         description='packaging',
322         descriptionDone='packaged'))
323     if rsync:
324         f.addStep(rsync_step(id, branch, rsync_script))
325     else:
326         f.addStep(FileUpload(name='upload',
327                              workersrc='buildbot_upload.zip',
328                              masterdest=filename,
329                              maxsize=180 * 1024 * 1024,
330                              workdir='install'))
331     f.addStep(MasterShellCommand(name='unpack',
332                                  command=['python3.6',
333                                           unpack_script,
334                                           filename],
335                                  description='unpacking',
336                                  descriptionDone='unpacked'))
337     return f
338
339
340 # Builders
341
342 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
343 add_builder(c, 'linux_glibc217_x86_64_cmake', '', generic_builder, hour=1)
344 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
345 # Visual Studio 2015 (vc14)
346 add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=1)
347
348 # HORIZONS
349 from datetime import timedelta
350
351 c['changeHorizon'] = 300
352
353 # Configure a janitor which will delete all logs older than one month,
354 # and will run on sundays at noon.
355 c['configurators'] = [util.JanitorConfigurator(
356     logHorizon=timedelta(weeks=4),
357     hour=12,
358     dayOfWeek=6)]
359
360
361 # WWW
362 c['www'] = dict(port=8010,
363                 plugins={'console_view': {},
364                          'grid_view': {},
365                          'waterfall_view': {}})
366
367 # Access
368 from buildbot.www.authz.roles import RolesFromOwner
369 from buildbot.www.authz.roles import RolesFromBase
370
371 class StrangerRoles(RolesFromBase):
372     def getRolesFromUser(self, userDetails):
373         return ['stranger']
374
375
376 authz = util.Authz(
377     stringsMatcher=util.fnmatchStrMatcher,  # simple matcher with '*' glob character
378     # stringsMatcher = util.reStrMatcher,   # if you prefer regular expressions
379     allowRules=[
380         # admins can do anything,
381         # defaultDeny=False: if user does not have the admin role, we continue parsing rules
382         util.AnyEndpointMatcher(role="admins", defaultDeny=False),
383
384         # Defaulting old config, everyone can stop build.
385         util.StopBuildEndpointMatcher(role="stranger"),
386         util.ForceBuildEndpointMatcher(role="stranger"),
387         util.RebuildBuildEndpointMatcher(role="stranger"),
388         util.EnableSchedulerEndpointMatcher(role="admins"),
389
390         # if future Buildbot implement new control, we are safe with this last rule
391         # NOTE: This prevents us from cancelling queue, so disabled for now.
392         # util.AnyControlEndpointMatcher(role="admins")
393     ],
394     roleMatchers=[
395         RolesFromOwner(role="owner"),
396         StrangerRoles(),
397     ])
398 c['www']['authz'] = authz
399
400 # PROJECT IDENTITY
401 c['projectName'] = "Blender"
402 c['projectURL'] = "https://www.blender.org"
403
404 # Buildbot information
405 c['buildbotURL'] = "https://builder.blender.org/admin/"
406 c['buildbotNetUsageData'] = 'basic'
407
408 # Various
409 c['db_url'] = "sqlite:///state.sqlite"
410
411 c['title'] = "Blender"
412 c['titleURL'] = "https://builder.blender.org/"
413
414 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
415 c['buildbotNetUsageData'] = None