Remove 291
[blender-buildbot.git] / master.cfg
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 # -*- python -*-
20 # ex: set syntax=python:
21
22 # <pep8 compliant>
23
24
25 from buildbot.www.authz.roles import RolesFromBase
26 from buildbot.www.authz.roles import RolesFromOwner
27 from buildbot.steps.master import MasterShellCommand
28 from buildbot.steps.transfer import FileUpload
29 from buildbot.steps.shell import Test
30 from buildbot.steps.shell import Compile
31 from buildbot.steps.shell import ShellCommand
32 from buildbot.process.properties import Interpolate
33 from buildbot.process.factory import BuildFactory
34 from buildbot.plugins import steps, util
35 from buildbot.config import BuilderConfig
36 from buildbot.schedulers import timed, forcesched
37 from buildbot.schedulers.basic import SingleBranchScheduler
38 from buildbot.schedulers.filter import ChangeFilter
39 from buildbot.changes.gitpoller import GitPoller
40 from buildbot.changes.svnpoller import SVNPoller
41 from buildbot.worker import Worker
42
43 from datetime import timedelta
44
45 # NOTE: We load the workers and their passwords from a separator file, so we can
46 # have # this one in Git.
47 import master_private
48
49 # Dictionary that the buildmaster pays attention to.
50 c = BuildmasterConfig = {}
51
52 # Project identity.
53 c['projectName'] = 'Blender'
54 c['projectURL'] = 'https://www.blender.org/'
55 c['title'] = 'Blender'
56 c['titleURL'] = 'https://builder.blender.org/'
57
58 # Buildbot information.
59 c['buildbotURL'] = 'https://builder.blender.org/admin/'
60 c['buildbotNetUsageData'] = 'basic'
61
62 # Various.
63 c['db_url'] = 'sqlite:///state.sqlite'
64
65 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
66 c['buildbotNetUsageData'] = None
67
68 ################################################################################
69 # BUILD WORKERS
70
71 c['workers'] = []
72 for worker in master_private.workers:
73     c['workers'].append(
74         Worker(worker['name'], worker['password'], max_builds=1))
75
76 # TCP port through which workers connect
77 c['protocols'] = {
78     'pb': {
79         'port': 'tcp:{}'.format(9989)
80     }
81 }
82
83 ################################################################################
84 # CODEBASES
85 #
86 # Allow to control separately things like branches for each repo and submodules.
87
88 all_repositories = {
89     r'git://git.blender.org/blender.git': 'blender',
90     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
91 }
92
93
94 def codebaseGenerator(chdict):
95     return all_repositories[chdict['repository']]
96
97
98 c['codebaseGenerator'] = codebaseGenerator
99
100
101 ################################################################################
102 # SCHEDULERS
103 #
104 # Decide how to react to incoming changes.
105
106
107 c['schedulers'] = []
108
109
110 def schedule_force_build(name, branch):
111     """
112     Makes it possible to have "Force Build" for the given builder.
113     Makes sure only reasonable subset of properties are exposed.
114     """
115     if branch != '':
116         branch_parameter = forcesched.FixedParameter(
117             name='branch',
118             default=branch,
119             hide=True)
120     else:
121         branch_parameter = forcesched.StringParameter(
122             name='branch',
123             label='Branch:',
124             default='custom-branch-name-here',
125             regex=r'^[a-zA-Z0-9][A-Za-z0-9\._-]*$')
126
127     c['schedulers'].append(forcesched.ForceScheduler(
128         name='force_' + name,
129         buttonName='Force Build',
130         builderNames=[name],
131         codebases=[forcesched.CodebaseParameter(
132             codebase='blender',
133             branch=branch_parameter,
134             # Hide revision. We don't want to allow anyone to overwrite the
135             # master build with an older version. Could be added back once we
136             # have authentication.
137             revision=forcesched.FixedParameter(
138                 name='revision',
139                 default='',
140                 hide=True),
141             repository=forcesched.FixedParameter(
142                 name='repository',
143                 default='',
144                 hide=True),
145             project=forcesched.FixedParameter(
146                 name='project',
147                 default='',
148                 hide=True)),
149         ]))
150
151
152 def schedule_nightly_build(name, branch, hour, minute=0):
153     """
154     Creates scheduler for nightly builds for a given builder.
155     """
156     scheduler_name = f'nightly_{name} {branch}'
157     c['schedulers'].append(timed.Nightly(
158         name=scheduler_name,
159         codebases={
160             'blender': {'repository': '',
161                         'branch': branch}},
162         branch=branch,
163         builderNames=[name],
164         hour=hour,
165         minute=minute))
166
167
168 def schedule_change_build(name, branch):
169     """
170     Creates scheduler for building on changes.
171     This will not package and upload the build.
172     """
173     scheduler_name = f'change_{name} {branch}'
174     c['schedulers'].append(SingleBranchScheduler(
175         name=scheduler_name,
176         codebases={
177             'blender': {'repository': '',
178                         'branch': branch}},
179         builderNames=[name],
180         treeStableTimer=120,
181         change_filter=ChangeFilter(project=['blender'], branch=branch),
182         properties={'skip_upload': True, 'skip_codesign': True}))
183
184
185 ################################################################################
186 # BUILDERS
187 #
188 # The 'builders' list defines the Builders, which tell Buildbot how to
189 # perform a build: what steps, and which workers can execute them.
190 # Note that any particular build will only take place on one worker.
191
192 c['builders'] = []
193 builders_all_branches = set()
194
195
196 # Add builder utility.
197
198 def add_builder(c, name, platforms, factory, branch='', hour=3, minute=0, use_nightly_builder=True):
199     if branch != '':
200         builders_all_branches.add(branch)
201
202     for platform in platforms:
203         workernames = []
204         builder_name = f'{name}_{platform}'
205
206         for worker in master_private.workers:
207             if platform == worker['platform']:
208                 workernames.append(worker['name'])
209                 builder_name = f"{worker['platform_short']}_{name}"
210
211         if workernames:
212             f = factory(builder_name, branch)
213             c['builders'].append(BuilderConfig(name=builder_name,
214                                                workernames=workernames,
215                                                factory=f,
216                                                tags=['blender']))
217
218             if use_nightly_builder:
219                 schedule_nightly_build(builder_name, branch, hour, minute)
220                 schedule_change_build(builder_name, branch)
221             schedule_force_build(builder_name, branch)
222
223
224 # Common steps.
225
226 def git_step(branch=''):
227     if branch:
228         return steps.Git(name='blender.git',
229                          repourl='git://git.blender.org/blender.git',
230                          mode='incremental',
231                          branch=branch,
232                          codebase='blender',
233                          workdir='blender.git',
234                          submodules=True)
235     else:
236         return steps.Git(name='blender.git',
237                          repourl='git://git.blender.org/blender.git',
238                          mode='incremental',
239                          codebase='blender',
240                          workdir='blender.git',
241                          submodules=True)
242
243
244 # Generic builder.
245
246 def do_upload(step):
247     return not step.hasProperty('skip_upload')
248
249 @util.renderer
250 def script_command(props, script, id, branch):
251     # NOTE: On Windows never includes major version in the executable name,
252     # so Python 3 will have be 'python.exe'.
253     if id.startswith('win'):
254         python_command = 'python'
255     else:
256         python_command = 'python3'
257
258     git_branch = branch or Interpolate('%(src:blender:branch)s')
259     args = [python_command, script, id, git_branch]
260
261     if not props.hasProperty('skip_codesign'):
262         args += ['--codesign']
263
264     return args
265
266 def generic_builder(id, branch=''):
267     # TODO(sergey): Consider using pathlib.
268     filename = f'uploaded/buildbot_upload_{id}.zip'
269     update_script = '../blender.git/build_files/buildbot/worker_update.py'
270     compile_script = '../blender.git/build_files/buildbot/worker_compile.py'
271     test_script = '../blender.git/build_files/buildbot/worker_test.py'
272     pack_script = '../blender.git/build_files/buildbot/worker_pack.py'
273     unpack_script = 'master_unpack.py'
274
275     f = BuildFactory()
276     f.addStep(git_step(branch))
277
278     f.addStep(ShellCommand(
279         name='submodules and libraries update',
280         command=script_command.withArgs(update_script, id, branch),
281         description='updating',
282         descriptionDone='updated',
283         haltOnFailure=True))
284
285     f.addStep(Compile(
286         command=script_command.withArgs(compile_script, id, branch),
287         timeout=3600))
288     f.addStep(Test(
289         command=script_command.withArgs(test_script, id, branch)))
290     f.addStep(ShellCommand(
291         name='package',
292         command=script_command.withArgs(pack_script, id, branch),
293         description='packaging',
294         descriptionDone='packaged',
295         haltOnFailure=True,
296         doStepIf=do_upload))
297
298     f.addStep(FileUpload(name='upload',
299                          workersrc='buildbot_upload.zip',
300                          masterdest=filename,
301                          maxsize=500 * 1024 * 1024,
302                          workdir='install',
303                          doStepIf=do_upload))
304     f.addStep(MasterShellCommand(name='unpack',
305                                  command=['python3.7',
306                                           unpack_script,
307                                           filename],
308                                  description='unpacking',
309                                  descriptionDone='unpacked',
310                                  doStepIf=do_upload))
311     return f
312
313
314 ################################################################################
315 # Builders
316
317 add_builder(c,
318             'master',
319             ['windows', 'macOS_10_15', 'linux_centos7'],
320             generic_builder,
321             branch='master',
322             hour=1,
323             use_nightly_builder=True)
324 add_builder(c,
325             '292',
326             ['windows', 'macOS_10_15', 'linux_centos7'],
327             generic_builder,
328             branch='blender-v2.92-release',
329             hour=1,
330             use_nightly_builder=False)
331 add_builder(c,
332             'lts_283',
333             ['windows', 'macOS_10_13', 'linux_centos7'],
334             generic_builder,
335             branch='blender-v2.83-release',
336             hour=1,
337             use_nightly_builder=False)
338 add_builder(c,
339             'custom_branch',
340             ['windows', 'macOS_10_15', 'linux_centos7'],
341             generic_builder,
342             branch='',
343             hour=1,
344             use_nightly_builder=False)
345
346 ################################################################################
347 # CHANGE SOURCES
348
349 c['change_source'] = GitPoller(repourl='git://git.blender.org/blender.git',
350                                pollinterval=120,
351                                project='blender',
352                                branches=list(builders_all_branches))
353
354 ################################################################################
355 # HORIZONS
356
357 c['changeHorizon'] = 300
358
359 # Configure a janitor which will delete all logs older than one month,
360 # and will run on sundays at noon.
361 c['configurators'] = [util.JanitorConfigurator(
362     logHorizon=timedelta(weeks=4),
363     hour=12,
364     dayOfWeek=6)]
365
366
367 ################################################################################
368 # WWW
369
370 c['www'] = dict(port=8010,
371                 plugins={'console_view': {},
372                          'grid_view': {},
373                          'waterfall_view': {}})
374
375 ################################################################################
376 # Access
377
378
379 class StrangerRoles(RolesFromBase):
380     def getRolesFromUser(self, userDetails):
381         return ['stranger']
382
383
384 authz = util.Authz(
385     # Simple matcher with '*' glob character.
386     # Could be util.reStrMatcher if regular expressions are preferred.
387     stringsMatcher=util.fnmatchStrMatcher,
388
389     allowRules=[
390         # Admins can do anything,
391         #
392         # defaultDeny=False: if user does not have the admin role, we continue
393         # parsing rules
394         util.AnyEndpointMatcher(role='admins', defaultDeny=False),
395
396         # Defaulting old config, everyone can stop build.
397         util.StopBuildEndpointMatcher(role='stranger'),
398         util.ForceBuildEndpointMatcher(role='stranger'),
399         util.RebuildBuildEndpointMatcher(role='stranger'),
400         util.EnableSchedulerEndpointMatcher(role='admins'),
401
402         # If future Buildbot implement new control, we are safe with this last
403         # rule.
404         #
405         # NOTE: This prevents us from cancelling queue, so disabled for now.
406         # util.AnyControlEndpointMatcher(role='admins')
407     ],
408     roleMatchers=[
409         RolesFromOwner(role='owner'),
410         StrangerRoles(),
411     ])
412 c['www']['authz'] = authz