Use same branch for contrib addons as for official addons
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
11
12 # List of branches availble for addons branch selection.
13 ADDONS_BRANCHES = ["master", "blender2.8"]
14
15 # List of branches availble for libraries branch selection.
16 LIB_BRANCHES = ["trunk"]
17
18 # Dictionary that the buildmaster pays attention to.
19 c = BuildmasterConfig = {}
20
21 # BUILD WORKERS
22 #
23 # We load the slaves and their passwords from a separator file, so we can have
24 # this one in SVN.
25
26 from buildbot.worker import Worker
27 import master_private
28
29 c['workers'] = []
30 for slave in master_private.slaves:
31     c['workers'].append(Worker(slave['name'], slave['password']))
32
33 # TCP port through which slaves connect
34 c['protocols'] = {
35     "pb": {
36         "port": "tcp:{}".format(9989)
37     }
38 }
39 # CHANGE SOURCES
40
41 from buildbot.changes.svnpoller import SVNPoller
42 from buildbot.changes.gitpoller import GitPoller
43
44 c['change_source'] = GitPoller('git://git.blender.org/blender.git',
45                                pollinterval=1200)
46
47 # CODEBASES
48 #
49 # Allow to control separately things like branches for each repo and submodules.
50
51 all_repositories = {
52     r'git://git.blender.org/blender.git': 'blender',
53     r'git://git.blender.org/blender-translations.git': 'blender-translations',
54     r'git://git.blender.org/blender-addons.git': 'blender-addons',
55     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
56     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
57     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
58 }
59
60
61 def codebaseGenerator(chdict):
62     return all_repositories[chdict['repository']]
63
64
65 c['codebaseGenerator'] = codebaseGenerator
66
67
68 # SCHEDULERS
69 #
70 # Decide how to react to incoming changes.
71
72 from buildbot.schedulers import timed, forcesched
73
74 c['schedulers'] = []
75
76
77 def schedule_force_build(name):
78     """
79     Makes it possible to have "Force Build" for the given builder.
80     Makes sure only reasonabel subset of properties are exposed.
81     """
82     c['schedulers'].append(forcesched.ForceScheduler(
83         name='force_' + name,
84         buttonName="Force Build",
85         builderNames=[name],
86         codebases=[forcesched.CodebaseParameter(
87                 codebase="blender",
88                 branch=forcesched.ChoiceStringParameter(
89                     name="branch",
90                     choices=FORCE_SCHEDULE_BRANCHES,
91                     default="master"),
92                 # Do not hide revision, can be handy!
93                 repository=forcesched.FixedParameter(
94                     name="repository",
95                     default="",
96                     hide=True),
97                 project=forcesched.FixedParameter(
98                     name="project",
99                     default="",
100                     hide=True)),
101                    # For now, hide other codebases.
102                    forcesched.CodebaseParameter(
103                        hide=True,
104                        codebase="blender-translations"),
105                    forcesched.CodebaseParameter(
106                        codebase="blender-addons",
107                        branch=forcesched.ChoiceStringParameter(
108                            name="branch",
109                            choices=ADDONS_BRANCHES,
110                            default="master"),
111                        repository=forcesched.FixedParameter(name="repository",
112                                                             default="",
113                                                             hide=True),
114                        project=forcesched.FixedParameter(name="project",
115                                                          default="",
116                                                          hide=True),
117                        revision=forcesched.FixedParameter(name="revision",
118                                                           default="",
119                                                           hide=True),
120                    ),
121                    forcesched.CodebaseParameter(
122                        codebase="blender-addons-contrib",
123                        branch=forcesched.ChoiceStringParameter(
124                            name="branch",
125                            choices=ADDONS_BRANCHES,
126                            default="master"),
127                        repository=forcesched.FixedParameter(name="repository",
128                                                             default="",
129                                                             hide=True),
130                        project=forcesched.FixedParameter(name="project",
131                                                          default="",
132                                                          hide=True),
133                        revision=forcesched.FixedParameter(name="revision",
134                                                           default="",
135                                                           hide=True),
136                    ),
137                    forcesched.CodebaseParameter(
138                        hide=True,
139                        codebase="blender-dev-tools"),
140                    forcesched.CodebaseParameter(
141                        codebase="lib svn",
142                        branch=forcesched.ChoiceStringParameter(
143                            name="branch",
144                            choices=LIB_BRANCHES,
145                            default="trunk"),
146                        repository=forcesched.FixedParameter(name="repository",
147                                                             default="",
148                                                             hide=True),
149                        project=forcesched.FixedParameter(name="project",
150                                                          default="",
151                                                          hide=True),
152                        revision=forcesched.FixedParameter(name="revision",
153                                                           default="",
154                                                           hide=True),
155                    ),
156         ],
157         properties=[]))
158
159
160 def schedule_nightly_build(name, hour, minute=0):
161     """
162     Creates scheduler for nightly builds for a given builder.
163     """
164     for current_branch in NIGHT_SCHEDULE_BRANCHES:
165         scheduler_name = "nightly_" + name
166         if current_branch:
167             scheduler_name += ' ' + current_branch
168         # Use special addons submodule branch when building blender2.8 branch.
169         addons_branch = "master"
170         if current_branch == "blender2.8":
171             addons_branch = "blender2.8"
172         c['schedulers'].append(timed.Nightly(
173             name=scheduler_name,
174             codebases={
175                 "blender": {"repository": "",
176                             "branch": current_branch},
177                 "blender-translations": {"repository": "",
178                                          "branch": "master"},
179                 "blender-addons": {"repository": "",
180                                    "branch": addons_branch},
181                 "blender-addons-contrib": {"repository": "",
182                                            "branch": addons_branch},
183                 "blender-dev-tools": {"repository": "",
184                                       "branch": "master"},
185                 "lib svn": {"repository": "",
186                             "branch": "trunk"}},
187             branch=current_branch,
188             builderNames=[name],
189             hour=hour,
190             minute=minute))
191
192
193 # BUILDERS
194 #
195 # The 'builders' list defines the Builders, which tell Buildbot how to
196 # perform a build: what steps, and which slaves can execute them.
197 # Note that any particular build will only take place on one slave.
198
199 from buildbot.config import BuilderConfig
200 from buildbot.plugins import steps, util
201 from buildbot.process.factory import BuildFactory
202 from buildbot.process.properties import Interpolate
203 from buildbot.steps.shell import ShellCommand
204 from buildbot.steps.shell import Compile
205 from buildbot.steps.shell import Test
206 from buildbot.steps.transfer import FileUpload
207 from buildbot.steps.master import MasterShellCommand
208
209 # add builder utility
210
211 c['builders'] = []
212 buildernames = []
213
214
215 def add_builder(c, name, libdir, factory, branch='',
216                 rsync=False, hour=3, minute=0):
217     workernames = []
218
219     for slave in master_private.slaves:
220         if name in slave['builders']:
221             workernames.append(slave['name'])
222
223     if workernames:
224         f = factory(name, libdir, branch, rsync)
225         c['builders'].append(BuilderConfig(name=name,
226                                            workernames=workernames,
227                                            factory=f,
228                                            tags=['blender']))
229         buildernames.append(name)
230
231         schedule_nightly_build(name, hour, minute)
232         schedule_force_build(name)
233
234
235 # common steps
236
237 def git_submodule_step(submodule):
238     return steps.Git(name=submodule + '.git',
239                      repourl='git://git.blender.org/' + submodule + '.git',
240                      mode='incremental',
241                      codebase=submodule,
242                      workdir=submodule + '.git')
243
244
245 def git_step(branch=''):
246     if branch:
247         return steps.Git(name='blender.git',
248                          repourl='git://git.blender.org/blender.git',
249                          mode='incremental',
250                          branch=branch,
251                          codebase='blender',
252                          workdir='blender.git',
253                          submodules=True)
254     else:
255         return steps.Git(name='blender.git',
256                          repourl='git://git.blender.org/blender.git',
257                          mode='incremental',
258                          codebase='blender',
259                          workdir='blender.git',
260                          submodules=True)
261
262
263 def git_submodules_update():
264     command = ['git', 'submodule', 'update', '--remote']
265     return ShellCommand(name='Submodules Update',
266                         command=command,
267                         description='updating',
268                         descriptionDone='up to date',
269                         workdir='blender.git')
270
271
272 def lib_svn_step(dir):
273     # TODO(sergey): For some reason interpolation is always giving empty branch.
274     # lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
275     lib_repo = 'https://svn.blender.org/svnroot/bf-blender/trunk/lib/'
276     return steps.SVN(name='lib svn',
277                      repourl=util.Interpolate(lib_repo + dir),
278                      codebase='lib svn',
279                      mode='incremental',
280                      workdir='lib/' + dir)
281
282
283 def rsync_step(id, branch, rsync_script):
284     return ShellCommand(name='rsync',
285                         command=['python', rsync_script, id, branch],
286                         description='uploading',
287                         descriptionDone='uploaded',
288                         workdir='install')
289
290
291 # generic builder
292
293 def generic_builder(id, libdir='', branch='', rsync=False):
294     filename = 'uploaded/buildbot_upload_' + id + '.zip'
295     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
296     test_script = '../blender.git/build_files/buildbot/slave_test.py'
297     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
298     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
299     unpack_script = 'master_unpack.py'
300
301     f = BuildFactory()
302     if libdir != '':
303         f.addStep(lib_svn_step(libdir))
304
305     for submodule in ('blender-translations',
306                       'blender-addons',
307                       'blender-addons-contrib',
308                       'blender-dev-tools'):
309         f.addStep(git_submodule_step(submodule))
310
311     f.addStep(git_step(branch))
312     f.addStep(git_submodules_update())
313
314     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
315     f.addStep(Test(command=['python', test_script, id]))
316     f.addStep(ShellCommand(
317         name='package',
318         command=['python',
319                  pack_script,
320                  id,
321                  branch or Interpolate('%(src:blender:branch)s')],
322         description='packaging',
323         descriptionDone='packaged'))
324     if rsync:
325         f.addStep(rsync_step(id, branch, rsync_script))
326     else:
327         f.addStep(FileUpload(name='upload',
328                              workersrc='buildbot_upload.zip',
329                              masterdest=filename,
330                              maxsize=180 * 1024 * 1024,
331                              workdir='install'))
332     f.addStep(MasterShellCommand(name='unpack',
333                                  command=['python2.7',
334                                           unpack_script,
335                                           filename],
336                                  description='unpacking',
337                                  descriptionDone='unpacked'))
338     return f
339
340
341 # Builders
342
343 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
344 add_builder(c, 'linux_glibc224_i686_cmake', '', generic_builder, hour=2)
345 add_builder(c, 'linux_glibc224_x86_64_cmake', '', generic_builder, hour=1)
346 # NOTE: Visual Studio 2017 (vc15) is using libraries folder from
347 # Visual Studio 2015 (vc14)
348 add_builder(c, 'win32_cmake_vs2017', 'windows_vc14', generic_builder, hour=1)
349 add_builder(c, 'win64_cmake_vs2017', 'win64_vc14', generic_builder, hour=2)
350
351 # HORIZONS
352 from datetime import timedelta
353
354 c['changeHorizon'] = 300
355
356 # Configure a janitor which will delete all logs older than one month,
357 # and will run on sundays at noon.
358 c['configurators'] = [util.JanitorConfigurator(
359     logHorizon=timedelta(weeks=4),
360     hour=12,
361     dayOfWeek=6)]
362
363
364 # WWW
365 c['www'] = dict(port=8010,
366                 plugins={'console_view': {},
367                          'grid_view': {},
368                          'waterfall_view': {}})
369
370 # Access
371 from buildbot.www.authz.roles import RolesFromOwner
372 from buildbot.www.authz.roles import RolesFromBase
373
374 class StrangerRoles(RolesFromBase):
375     def getRolesFromUser(self, userDetails):
376         return ['stranger']
377
378
379 authz = util.Authz(
380     stringsMatcher=util.fnmatchStrMatcher,  # simple matcher with '*' glob character
381     # stringsMatcher = util.reStrMatcher,   # if you prefer regular expressions
382     allowRules=[
383         # admins can do anything,
384         # defaultDeny=False: if user does not have the admin role, we continue parsing rules
385         util.AnyEndpointMatcher(role="admins", defaultDeny=False),
386
387         # Defaulting old config, everyone can stop build.
388         util.StopBuildEndpointMatcher(role="stranger"),
389         util.ForceBuildEndpointMatcher(role="stranger"),
390         util.RebuildBuildEndpointMatcher(role="stranger"),
391         util.EnableSchedulerEndpointMatcher(role="admins"),
392
393         # if future Buildbot implement new control, we are safe with this last rule
394         # NOTE: This prevents us from cancelling queue, so disabled for now.
395         # util.AnyControlEndpointMatcher(role="admins")
396     ],
397     roleMatchers=[
398         RolesFromOwner(role="owner"),
399         StrangerRoles(),
400     ])
401 c['www']['authz'] = authz
402
403 # PROJECT IDENTITY
404 c['projectName'] = "Blender"
405 c['projectURL'] = "https://www.blender.org"
406
407 # Buildbot information
408 c['buildbotURL'] = "https://builder.blender.org/admin/"
409 c['buildbotNetUsageData'] = 'basic'
410
411 # Various
412 c['db_url'] = "sqlite:///state.sqlite"
413
414 c['title'] = "Blender"
415 c['titleURL'] = "https://builder.blender.org/"
416
417 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
418 c['buildbotNetUsageData'] = None