8659f9a5827a6bdc91828de332f09187fe98d933
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = ["master", "blender2.7"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.7", "experimental-build"]
11
12 # List of branches availble for addons branch selection.
13 ADDONS_BRANCHES = ["master", "blender2.7"]
14
15 # List of branches availble for libraries branch selection.
16 LIB_BRANCHES = ["trunk"]
17
18 # Dictionary that the buildmaster pays attention to.
19 c = BuildmasterConfig = {}
20
21 # BUILD WORKERS
22 #
23 # We load the slaves and their passwords from a separator file, so we can have
24 # this one in SVN.
25
26 from buildbot.worker import Worker
27 import master_private
28
29 c['workers'] = []
30 for slave in master_private.slaves:
31     c['workers'].append(Worker(slave['name'], slave['password']))
32
33 # TCP port through which slaves connect
34 c['protocols'] = {
35     "pb": {
36         "port": "tcp:{}".format(9989)
37     }
38 }
39 # CHANGE SOURCES
40
41 from buildbot.changes.svnpoller import SVNPoller
42 from buildbot.changes.gitpoller import GitPoller
43
44 c['change_source'] = GitPoller('git://git.blender.org/blender.git',
45                                pollinterval=1200)
46
47 # CODEBASES
48 #
49 # Allow to control separately things like branches for each repo and submodules.
50
51 all_repositories = {
52     r'git://git.blender.org/blender.git': 'blender',
53     r'git://git.blender.org/blender-translations.git': 'blender-translations',
54     r'git://git.blender.org/blender-addons.git': 'blender-addons',
55     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
56     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
57     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
58 }
59
60
61 def codebaseGenerator(chdict):
62     return all_repositories[chdict['repository']]
63
64
65 c['codebaseGenerator'] = codebaseGenerator
66
67
68 # SCHEDULERS
69 #
70 # Decide how to react to incoming changes.
71
72 from buildbot.schedulers import timed, forcesched
73
74 c['schedulers'] = []
75
76
77 def schedule_force_build(name):
78     """
79     Makes it possible to have "Force Build" for the given builder.
80     Makes sure only reasonabel subset of properties are exposed.
81     """
82     c['schedulers'].append(forcesched.ForceScheduler(
83         name='force_' + name,
84         buttonName="Force Build",
85         builderNames=[name],
86         codebases=[forcesched.CodebaseParameter(
87                 codebase="blender",
88                 branch=forcesched.ChoiceStringParameter(
89                     name="branch",
90                     choices=FORCE_SCHEDULE_BRANCHES,
91                     default="master"),
92                 # Do not hide revision, can be handy!
93                 repository=forcesched.FixedParameter(
94                     name="repository",
95                     default="",
96                     hide=True),
97                 project=forcesched.FixedParameter(
98                     name="project",
99                     default="",
100                     hide=True)),
101                    # For now, hide other codebases.
102                    forcesched.CodebaseParameter(
103                        hide=True,
104                        codebase="blender-translations"),
105                    forcesched.CodebaseParameter(
106                        codebase="blender-addons",
107                        branch=forcesched.ChoiceStringParameter(
108                            name="branch",
109                            choices=ADDONS_BRANCHES,
110                            default="master"),
111                        repository=forcesched.FixedParameter(name="repository",
112                                                             default="",
113                                                             hide=True),
114                        project=forcesched.FixedParameter(name="project",
115                                                          default="",
116                                                          hide=True),
117                        revision=forcesched.FixedParameter(name="revision",
118                                                           default="",
119                                                           hide=True),
120                    ),
121                    forcesched.CodebaseParameter(
122                        codebase="blender-addons-contrib",
123                        branch=forcesched.ChoiceStringParameter(
124                            name="branch",
125                            choices=ADDONS_BRANCHES,
126                            default="master"),
127                        repository=forcesched.FixedParameter(name="repository",
128                                                             default="",
129                                                             hide=True),
130                        project=forcesched.FixedParameter(name="project",
131                                                          default="",
132                                                          hide=True),
133                        revision=forcesched.FixedParameter(name="revision",
134                                                           default="",
135                                                           hide=True),
136                    ),
137                    forcesched.CodebaseParameter(
138                        hide=True,
139                        codebase="blender-dev-tools"),
140                    forcesched.CodebaseParameter(
141                        codebase="lib svn",
142                        branch=forcesched.ChoiceStringParameter(
143                            name="branch",
144                            choices=LIB_BRANCHES,
145                            default="trunk"),
146                        repository=forcesched.FixedParameter(name="repository",
147                                                             default="",
148                                                             hide=True),
149                        project=forcesched.FixedParameter(name="project",
150                                                          default="",
151                                                          hide=True),
152                        revision=forcesched.FixedParameter(name="revision",
153                                                           default="",
154                                                           hide=True),
155                    ),
156         ],
157         properties=[]))
158
159
160 def schedule_nightly_build(name, hour, minute=0):
161     """
162     Creates scheduler for nightly builds for a given builder.
163     """
164     for current_branch in NIGHT_SCHEDULE_BRANCHES:
165         scheduler_name = "nightly_" + name
166         if current_branch:
167             scheduler_name += ' ' + current_branch
168         # Use special addons submodule branch when building blender2.7 branch.
169         addons_branch = "master"
170         if current_branch == "blender2.7":
171             addons_branch = "blender2.7"
172         c['schedulers'].append(timed.Nightly(
173             name=scheduler_name,
174             codebases={
175                 "blender": {"repository": "",
176                             "branch": current_branch},
177                 "blender-translations": {"repository": "",
178                                          "branch": "master"},
179                 "blender-addons": {"repository": "",
180                                    "branch": addons_branch},
181                 "blender-addons-contrib": {"repository": "",
182                                            "branch": addons_branch},
183                 "blender-dev-tools": {"repository": "",
184                                       "branch": "master"},
185                 "lib svn": {"repository": "",
186                             "branch": "trunk"}},
187             branch=current_branch,
188             builderNames=[name],
189             hour=hour,
190             minute=minute))
191
192
193 # BUILDERS
194 #
195 # The 'builders' list defines the Builders, which tell Buildbot how to
196 # perform a build: what steps, and which slaves can execute them.
197 # Note that any particular build will only take place on one slave.
198
199 from buildbot.config import BuilderConfig
200 from buildbot.plugins import steps, util
201 from buildbot.process.factory import BuildFactory
202 from buildbot.process.properties import Interpolate
203 from buildbot.steps.shell import ShellCommand
204 from buildbot.steps.shell import Compile
205 from buildbot.steps.shell import Test
206 from buildbot.steps.transfer import FileUpload
207 from buildbot.steps.master import MasterShellCommand
208
209 # add builder utility
210
211 c['builders'] = []
212 buildernames = []
213
214
215 def add_builder(c, name, libdir, factory, branch='',
216                 rsync=False, hour=3, minute=0):
217     workernames = []
218
219     for slave in master_private.slaves:
220         if name in slave['builders']:
221             workernames.append(slave['name'])
222
223     if workernames:
224         f = factory(name, libdir, branch, rsync)
225         c['builders'].append(BuilderConfig(name=name,
226                                            workernames=workernames,
227                                            factory=f,
228                                            tags=['blender']))
229         buildernames.append(name)
230
231         schedule_nightly_build(name, hour, minute)
232         schedule_force_build(name)
233
234
235 # common steps
236
237 def git_submodule_step(submodule):
238     return steps.Git(name=submodule + '.git',
239                      repourl='git://git.blender.org/' + submodule + '.git',
240                      mode='incremental',
241                      codebase=submodule,
242                      workdir=submodule + '.git')
243
244
245 def git_step(branch=''):
246     if branch:
247         return steps.Git(name='blender.git',
248                          repourl='git://git.blender.org/blender.git',
249                          mode='incremental',
250                          branch=branch,
251                          codebase='blender',
252                          workdir='blender.git',
253                          submodules=True)
254     else:
255         return steps.Git(name='blender.git',
256                          repourl='git://git.blender.org/blender.git',
257                          mode='incremental',
258                          codebase='blender',
259                          workdir='blender.git',
260                          submodules=True)
261
262
263 def git_submodules_update():
264     command = ['git', 'submodule', 'update', '--remote']
265     return ShellCommand(name='Submodules Update',
266                         command=command,
267                         description='updating',
268                         descriptionDone='up to date',
269                         workdir='blender.git')
270
271
272 def lib_svn_step(dir):
273     # TODO(sergey): For some reason interpolation is always giving empty branch.
274     # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
275     lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
276     return steps.SVN(name='lib svn',
277                      repourl=util.Interpolate(lib_repo + dir),
278                      codebase='lib svn',
279                      mode='incremental',
280                      workdir='lib/' + dir)
281
282
283 def rsync_step(id, branch, rsync_script):
284     return ShellCommand(name='rsync',
285                         command=['python', rsync_script, id, branch],
286                         description='uploading',
287                         descriptionDone='uploaded',
288                         workdir='install')
289
290
291 # generic builder
292
293 def generic_builder(id, libdir='', branch='', rsync=False):
294     filename = 'uploaded/buildbot_upload_' + id + '.zip'
295     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
296     test_script = '../blender.git/build_files/buildbot/slave_test.py'
297     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
298     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
299     unpack_script = 'master_unpack.py'
300
301     f = BuildFactory()
302     if libdir != '':
303         f.addStep(lib_svn_step(libdir))
304
305     for submodule in ('blender-translations',
306                       'blender-addons',
307                       'blender-addons-contrib',
308                       'blender-dev-tools'):
309         f.addStep(git_submodule_step(submodule))
310
311     f.addStep(git_step(branch))
312     f.addStep(git_submodules_update())
313
314     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
315     f.addStep(Test(command=['python', test_script, id]))
316     f.addStep(ShellCommand(
317         name='package',
318         command=['python',
319                  pack_script,
320                  id,
321                  branch or Interpolate('%(src:blender:branch)s')],
322         description='packaging',
323         descriptionDone='packaged'))
324     if rsync:
325         f.addStep(rsync_step(id, branch, rsync_script))
326     else:
327         f.addStep(FileUpload(name='upload',
328                              workersrc='buildbot_upload.zip',
329                              masterdest=filename,
330                              maxsize=180 * 1024 * 1024,
331                              workdir='install'))
332     f.addStep(MasterShellCommand(name='unpack',
333                                  command=['python3.6',
334                                           unpack_script,
335                                           filename],
336                                  description='unpacking',
337                                  descriptionDone='unpacked'))
338     return f
339
340
341 # Builders
342
343 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
344
345 add_builder(c, 'linux_glibc224_i686_cmake', '', generic_builder, hour=2)
346 add_builder(c, 'linux_glibc224_x86_64_cmake', '', generic_builder, hour=1)
347
348 add_builder(c, 'linux_glibc217_x86_64_cmake', '', generic_builder, hour=1)
349
350 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
351 # Visual Studio 2015 (vc14)
352 add_builder(c, 'win32_cmake_vs2017', 'windows_vc14', generic_builder, hour=1)
353 add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=2)
354
355 # HORIZONS
356 from datetime import timedelta
357
358 c['changeHorizon'] = 300
359
360 # Configure a janitor which will delete all logs older than one month,
361 # and will run on sundays at noon.
362 c['configurators'] = [util.JanitorConfigurator(
363     logHorizon=timedelta(weeks=4),
364     hour=12,
365     dayOfWeek=6)]
366
367
368 # WWW
369 c['www'] = dict(port=8010,
370                 plugins={'console_view': {},
371                          'grid_view': {},
372                          'waterfall_view': {}})
373
374 # Access
375 from buildbot.www.authz.roles import RolesFromOwner
376 from buildbot.www.authz.roles import RolesFromBase
377
378 class StrangerRoles(RolesFromBase):
379     def getRolesFromUser(self, userDetails):
380         return ['stranger']
381
382
383 authz = util.Authz(
384     stringsMatcher=util.fnmatchStrMatcher,  # simple matcher with '*' glob character
385     # stringsMatcher = util.reStrMatcher,   # if you prefer regular expressions
386     allowRules=[
387         # admins can do anything,
388         # defaultDeny=False: if user does not have the admin role, we continue parsing rules
389         util.AnyEndpointMatcher(role="admins", defaultDeny=False),
390
391         # Defaulting old config, everyone can stop build.
392         util.StopBuildEndpointMatcher(role="stranger"),
393         util.ForceBuildEndpointMatcher(role="stranger"),
394         util.RebuildBuildEndpointMatcher(role="stranger"),
395         util.EnableSchedulerEndpointMatcher(role="admins"),
396
397         # if future Buildbot implement new control, we are safe with this last rule
398         # NOTE: This prevents us from cancelling queue, so disabled for now.
399         # util.AnyControlEndpointMatcher(role="admins")
400     ],
401     roleMatchers=[
402         RolesFromOwner(role="owner"),
403         StrangerRoles(),
404     ])
405 c['www']['authz'] = authz
406
407 # PROJECT IDENTITY
408 c['projectName'] = "Blender"
409 c['projectURL'] = "https://www.blender.org"
410
411 # Buildbot information
412 c['buildbotURL'] = "https://builder.blender.org/admin/"
413 c['buildbotNetUsageData'] = 'basic'
414
415 # Various
416 c['db_url'] = "sqlite:///state.sqlite"
417
418 c['title'] = "Blender"
419 c['titleURL'] = "https://builder.blender.org/"
420
421 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
422 c['buildbotNetUsageData'] = None