Remove old windows builders and add new linux builders
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
11
12 # List of branches availble for addons branch selection.
13 ADDONS_BRANCHES = ["master", "blender2.8"]
14
15 # List of branches availble for libraries branch selection.
16 LIB_BRANCHES = ["trunk"]
17
18 # Dictionary that the buildmaster pays attention to.
19 c = BuildmasterConfig = {}
20
21 # BUILD WORKERS
22 #
23 # We load the slaves and their passwords from a separator file, so we can have
24 # this one in SVN.
25
26 from buildbot.worker import Worker
27 import master_private
28
29 c['workers'] = []
30 for slave in master_private.slaves:
31     c['workers'].append(Worker(slave['name'], slave['password']))
32
33 # TCP port through which slaves connect
34 c['protocols'] = {
35     "pb": {
36         "port": "tcp:{}".format(9989)
37     }
38 }
39 # CHANGE SOURCES
40
41 from buildbot.changes.svnpoller import SVNPoller
42 from buildbot.changes.gitpoller import GitPoller
43
44 c['change_source'] = GitPoller('git://git.blender.org/blender.git',
45                                pollinterval=1200)
46
47 # CODEBASES
48 #
49 # Allow to control separately things like branches for each repo and submodules.
50
51 all_repositories = {
52     r'git://git.blender.org/blender.git': 'blender',
53     r'git://git.blender.org/blender-translations.git': 'blender-translations',
54     r'git://git.blender.org/blender-addons.git': 'blender-addons',
55     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
56     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
57     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
58 }
59
60
61 def codebaseGenerator(chdict):
62     return all_repositories[chdict['repository']]
63
64
65 c['codebaseGenerator'] = codebaseGenerator
66
67
68 # SCHEDULERS
69 #
70 # Decide how to react to incoming changes.
71
72 from buildbot.schedulers import timed, forcesched
73
74 c['schedulers'] = []
75
76
77 def schedule_force_build(name):
78     """
79     Makes it possible to have "Force Build" for the given builder.
80     Makes sure only reasonabel subset of properties are exposed.
81     """
82     c['schedulers'].append(forcesched.ForceScheduler(
83         name='force_' + name,
84         buttonName="Force Build",
85         builderNames=[name],
86         codebases=[forcesched.CodebaseParameter(
87                 codebase="blender",
88                 branch=forcesched.ChoiceStringParameter(
89                     name="branch",
90                     choices=FORCE_SCHEDULE_BRANCHES,
91                     default="master"),
92                 # Do not hide revision, can be handy!
93                 repository=forcesched.FixedParameter(
94                     name="repository",
95                     default="",
96                     hide=True),
97                 project=forcesched.FixedParameter(
98                     name="project",
99                     default="",
100                     hide=True)),
101                    # For now, hide other codebases.
102                    forcesched.CodebaseParameter(
103                        hide=True,
104                        codebase="blender-translations"),
105                    forcesched.CodebaseParameter(
106                        codebase="blender-addons",
107                        branch=forcesched.ChoiceStringParameter(
108                            name="branch",
109                            choices=ADDONS_BRANCHES,
110                            default="master"),
111                        repository=forcesched.FixedParameter(name="repository",
112                                                             default="",
113                                                             hide=True),
114                        project=forcesched.FixedParameter(name="project",
115                                                          default="",
116                                                          hide=True),
117                        revision=forcesched.FixedParameter(name="revision",
118                                                           default="",
119                                                           hide=True),
120                    ),
121                    forcesched.CodebaseParameter(
122                        hide=True,
123                        codebase="blender-addons-contrib"),
124                    forcesched.CodebaseParameter(
125                        hide=True,
126                        codebase="blender-dev-tools"),
127                    forcesched.CodebaseParameter(
128                        codebase="lib svn",
129                        branch=forcesched.ChoiceStringParameter(
130                            name="branch",
131                            choices=LIB_BRANCHES,
132                            default="trunk"),
133                        repository=forcesched.FixedParameter(name="repository",
134                                                             default="",
135                                                             hide=True),
136                        project=forcesched.FixedParameter(name="project",
137                                                          default="",
138                                                          hide=True),
139                        revision=forcesched.FixedParameter(name="revision",
140                                                           default="",
141                                                           hide=True),
142                    ),
143         ],
144         properties=[]))
145
146
147 def schedule_nightly_build(name, hour, minute=0):
148     """
149     Creates scheduler for nightly builds for a given builder.
150     """
151     for current_branch in NIGHT_SCHEDULE_BRANCHES:
152         scheduler_name = "nightly_" + name
153         if current_branch:
154             scheduler_name += ' ' + current_branch
155         # Use special addons submodule branch when building blender2.8 branch.
156         addons_branch = "master"
157         if current_branch == "blender2.8":
158             addons_branch = "blender2.8"
159         c['schedulers'].append(timed.Nightly(
160             name=scheduler_name,
161             codebases={
162                 "blender": {"repository": "",
163                             "branch": current_branch},
164                 "blender-translations": {"repository": "",
165                                          "branch": "master"},
166                 "blender-addons": {"repository": "",
167                                    "branch": addons_branch},
168                 "blender-addons-contrib": {"repository": "",
169                                            "branch": "master"},
170                 "blender-dev-tools": {"repository": "",
171                                       "branch": "master"},
172                 "lib svn": {"repository": "",
173                             "branch": "trunk"}},
174             branch=current_branch,
175             builderNames=[name],
176             hour=hour,
177             minute=minute))
178
179
180 # BUILDERS
181 #
182 # The 'builders' list defines the Builders, which tell Buildbot how to
183 # perform a build: what steps, and which slaves can execute them.
184 # Note that any particular build will only take place on one slave.
185
186 from buildbot.config import BuilderConfig
187 from buildbot.plugins import steps, util
188 from buildbot.process.factory import BuildFactory
189 from buildbot.process.properties import Interpolate
190 from buildbot.steps.shell import ShellCommand
191 from buildbot.steps.shell import Compile
192 from buildbot.steps.shell import Test
193 from buildbot.steps.transfer import FileUpload
194 from buildbot.steps.master import MasterShellCommand
195
196 # add builder utility
197
198 c['builders'] = []
199 buildernames = []
200
201
202 def add_builder(c, name, libdir, factory, branch='',
203                 rsync=False, hour=3, minute=0):
204     workernames = []
205
206     for slave in master_private.slaves:
207         if name in slave['builders']:
208             workernames.append(slave['name'])
209
210     if workernames:
211         f = factory(name, libdir, branch, rsync)
212         c['builders'].append(BuilderConfig(name=name,
213                                            workernames=workernames,
214                                            factory=f,
215                                            tags=['blender']))
216         buildernames.append(name)
217
218         schedule_nightly_build(name, hour, minute)
219         schedule_force_build(name)
220
221
222 # common steps
223
224 def git_submodule_step(submodule):
225     return steps.Git(name=submodule + '.git',
226                      repourl='git://git.blender.org/' + submodule + '.git',
227                      mode='incremental',
228                      codebase=submodule,
229                      workdir=submodule + '.git')
230
231
232 def git_step(branch=''):
233     if branch:
234         return steps.Git(name='blender.git',
235                          repourl='git://git.blender.org/blender.git',
236                          mode='incremental',
237                          branch=branch,
238                          codebase='blender',
239                          workdir='blender.git',
240                          submodules=True)
241     else:
242         return steps.Git(name='blender.git',
243                          repourl='git://git.blender.org/blender.git',
244                          mode='incremental',
245                          codebase='blender',
246                          workdir='blender.git',
247                          submodules=True)
248
249
250 def git_submodules_update():
251     command = ['git', 'submodule', 'update', '--remote']
252     return ShellCommand(name='Submodules Update',
253                         command=command,
254                         description='updating',
255                         descriptionDone='up to date',
256                         workdir='blender.git')
257
258
259 def lib_svn_step(dir):
260     # TODO(sergey): For some reason interpolation is always giving empty branch.
261     # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
262     lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
263     return steps.SVN(name='lib svn',
264                      repourl=util.Interpolate(lib_repo + dir),
265                      codebase='lib svn',
266                      mode='incremental',
267                      workdir='lib/' + dir)
268
269
270 def rsync_step(id, branch, rsync_script):
271     return ShellCommand(name='rsync',
272                         command=['python', rsync_script, id, branch],
273                         description='uploading',
274                         descriptionDone='uploaded',
275                         workdir='install')
276
277
278 # generic builder
279
280 def generic_builder(id, libdir='', branch='', rsync=False):
281     filename = 'uploaded/buildbot_upload_' + id + '.zip'
282     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
283     test_script = '../blender.git/build_files/buildbot/slave_test.py'
284     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
285     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
286     unpack_script = 'master_unpack.py'
287
288     f = BuildFactory()
289     if libdir != '':
290         f.addStep(lib_svn_step(libdir))
291
292     for submodule in ('blender-translations',
293                       'blender-addons',
294                       'blender-addons-contrib',
295                       'blender-dev-tools'):
296         f.addStep(git_submodule_step(submodule))
297
298     f.addStep(git_step(branch))
299     f.addStep(git_submodules_update())
300
301     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
302     f.addStep(Test(command=['python', test_script, id]))
303     f.addStep(ShellCommand(
304         name='package',
305         command=['python',
306                  pack_script,
307                  id,
308                  branch or Interpolate('%(src:blender:branch)s')],
309         description='packaging',
310         descriptionDone='packaged'))
311     if rsync:
312         f.addStep(rsync_step(id, branch, rsync_script))
313     else:
314         f.addStep(FileUpload(name='upload',
315                              workersrc='buildbot_upload.zip',
316                              masterdest=filename,
317                              maxsize=180 * 1024 * 1024,
318                              workdir='install'))
319     f.addStep(MasterShellCommand(name='unpack',
320                                  command=['python2.7',
321                                           unpack_script,
322                                           filename],
323                                  description='unpacking',
324                                  descriptionDone='unpacked'))
325     return f
326
327
328 # Builders
329
330 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
331 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=4)
332 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=3)
333 add_builder(c, 'linux_glibc224_i686_cmake', '', generic_builder, hour=2)
334 add_builder(c, 'linux_glibc224_x86_64_cmake', '', generic_builder, hour=1)
335 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
336 # Visual Studio 2015 (vc14)
337 add_builder(c, 'win32_cmake_vs2017', 'windows_vc14', generic_builder, hour=1)
338 add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=2)
339
340 # HORIZONS
341 from datetime import timedelta
342
343 c['changeHorizon'] = 300
344
345 # Configure a janitor which will delete all logs older than one month,
346 # and will run on sundays at noon.
347 c['configurators'] = [util.JanitorConfigurator(
348     logHorizon=timedelta(weeks=4),
349     hour=12,
350     dayOfWeek=6)]
351
352
353 # WWW
354 c['www'] = dict(port=8010,
355                 plugins={'console_view': {},
356                          'grid_view': {},
357                          'waterfall_view': {}})
358
359 # Access
360 from buildbot.www.authz.roles import RolesFromOwner
361 from buildbot.www.authz.roles import RolesFromBase
362
363 class StrangerRoles(RolesFromBase):
364     def getRolesFromUser(self, userDetails):
365         return ['stranger']
366
367
368 authz = util.Authz(
369     stringsMatcher=util.fnmatchStrMatcher,  # simple matcher with '*' glob character
370     # stringsMatcher = util.reStrMatcher,   # if you prefer regular expressions
371     allowRules=[
372         # admins can do anything,
373         # defaultDeny=False: if user does not have the admin role, we continue parsing rules
374         util.AnyEndpointMatcher(role="admins", defaultDeny=False),
375
376         # Defaulting old config, everyone can stop build.
377         util.StopBuildEndpointMatcher(role="stranger"),
378         util.ForceBuildEndpointMatcher(role="stranger"),
379         util.RebuildBuildEndpointMatcher(role="stranger"),
380         util.EnableSchedulerEndpointMatcher(role="admins"),
381
382         # if future Buildbot implement new control, we are safe with this last rule
383         # NOTE: This prevents us from cancelling queue, so disabled for now.
384         # util.AnyControlEndpointMatcher(role="admins")
385     ],
386     roleMatchers=[
387         RolesFromOwner(role="owner"),
388         StrangerRoles(),
389     ])
390 c['www']['authz'] = authz
391
392 # PROJECT IDENTITY
393 c['projectName'] = "Blender"
394 c['projectURL'] = "https://www.blender.org"
395
396 # Buildbot information
397 c['buildbotURL'] = "https://builder.blender.org/admin/"
398 c['buildbotNetUsageData'] = 'basic'
399
400 # Various
401 c['db_url'] = "sqlite:///state.sqlite"
402
403 c['title'] = "Blender"
404 c['titleURL'] = "https://builder.blender.org/"
405
406 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
407 c['buildbotNetUsageData'] = None