Cleanup and more plugins enabled
[blender-buildbot.git] / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 # <pep8 compliant>
5
6 # List of the branches being built automatically overnight
7 NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
8
9 # List of the branches available for force build
10 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
11
12 # List of branches availble for addons branch selection.
13 ADDONS_BRANCHES = ["master", "blender2.8"]
14
15 # Dictionary that the buildmaster pays attention to.
16 c = BuildmasterConfig = {}
17
18 # BUILD WORKERS
19 #
20 # We load the slaves and their passwords from a separator file, so we can have
21 # this one in SVN.
22
23 from buildbot.worker import Worker
24 import master_private
25
26 c['workers'] = []
27 for slave in master_private.slaves:
28     c['workers'].append(Worker(slave['name'], slave['password']))
29
30 # TCP port through which slaves connect
31 c['protocols'] = {
32     "pb": {
33         "port": "tcp:{}".format(9989)
34     }
35 }
36 # CHANGE SOURCES
37
38 from buildbot.changes.svnpoller import SVNPoller
39 from buildbot.changes.gitpoller import GitPoller
40
41 c['change_source'] = GitPoller('git://git.blender.org/blender.git',
42                                pollinterval=1200)
43
44 # CODEBASES
45 #
46 # Allow to control separately things like branches for each repo and submodules.
47
48 all_repositories = {
49     r'git://git.blender.org/blender.git': 'blender',
50     r'git://git.blender.org/blender-translations.git': 'blender-translations',
51     r'git://git.blender.org/blender-addons.git': 'blender-addons',
52     r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib',
53     r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools',
54     r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn',
55 }
56
57
58 def codebaseGenerator(chdict):
59     return all_repositories[chdict['repository']]
60
61
62 c['codebaseGenerator'] = codebaseGenerator
63
64
65 # SCHEDULERS
66 #
67 # Decide how to react to incoming changes.
68
69 from buildbot.schedulers import timed, forcesched
70
71 c['schedulers'] = []
72
73
74 def schedule_force_build(name):
75     """
76     Makes it possible to have "Force Build" for the given builder.
77     Makes sure only reasonabel subset of properties are exposed.
78     """
79     c['schedulers'].append(forcesched.ForceScheduler(
80         name='force_' + name,
81         buttonName="Force Build",
82         builderNames=[name],
83         codebases=[forcesched.CodebaseParameter(
84                 codebase="blender",
85                 branch=forcesched.ChoiceStringParameter(
86                     name="branch",
87                     choices=FORCE_SCHEDULE_BRANCHES,
88                     default="master"),
89                 # Do not hide revision, can be handy!
90                 repository=forcesched.FixedParameter(
91                     name="repository",
92                     default="",
93                     hide=True),
94                 project=forcesched.FixedParameter(
95                     name="project",
96                     default="",
97                     hide=True)),
98                    # For now, hide other codebases.
99                    forcesched.CodebaseParameter(
100                        hide=True,
101                        codebase="blender-translations"),
102                    forcesched.CodebaseParameter(
103                        codebase="blender-addons",
104                        branch=forcesched.ChoiceStringParameter(
105                            name="branch",
106                            choices=ADDONS_BRANCHES,
107                            default="master"),
108                        repository=forcesched.FixedParameter(name="repository",
109                                                             default="",
110                                                             hide=True),
111                        project=forcesched.FixedParameter(name="project",
112                                                          default="",
113                                                          hide=True),
114                        revision=forcesched.FixedParameter(name="revision",
115                                                           default="",
116                                                           hide=True),
117                    ),
118                    forcesched.CodebaseParameter(
119                        hide=True,
120                        codebase="blender-addons-contrib"),
121                    forcesched.CodebaseParameter(
122                        hide=True,
123                        codebase="blender-dev-tools"),
124                    forcesched.CodebaseParameter(
125                        hide=True,
126                        codebase="lib svn")],
127         properties=[]))
128
129
130 def schedule_nightly_build(name, hour, minute=0):
131     """
132     Creates scheduler for nightly builds for a given builder.
133     """
134     for current_branch in NIGHT_SCHEDULE_BRANCHES:
135         scheduler_name = "nightly_" + name
136         if current_branch:
137             scheduler_name += ' ' + current_branch
138         # Use special addons submodule branch when building blender2.8 branch.
139         addons_branch = "master"
140         if current_branch == "blender2.8":
141             addons_branch = "blender2.8"
142         c['schedulers'].append(timed.Nightly(
143             name=scheduler_name,
144             codebases={
145                 "blender": {"repository": ""},
146                 "blender-translations": {"repository": "",
147                                          "branch": "master"},
148                 "blender-addons": {"repository": "",
149                                    "branch": addons_branch},
150                 "blender-addons-contrib": {"repository": "",
151                                            "branch": "master"},
152                 "blender-dev-tools": {"repository": "",
153                                       "branch": "master"},
154                 "lib svn": {"repository": "",
155                             "branch": "trunk"}},
156             branch=current_branch,
157             builderNames=[name],
158             hour=hour,
159             minute=minute))
160
161
162 # BUILDERS
163 #
164 # The 'builders' list defines the Builders, which tell Buildbot how to
165 # perform a build: what steps, and which slaves can execute them.
166 # Note that any particular build will only take place on one slave.
167
168 from buildbot.config import BuilderConfig
169 from buildbot.plugins import steps, util
170 from buildbot.process.factory import BuildFactory
171 from buildbot.process.properties import Interpolate
172 from buildbot.steps.shell import ShellCommand
173 from buildbot.steps.shell import Compile
174 from buildbot.steps.shell import Test
175 from buildbot.steps.transfer import FileUpload
176 from buildbot.steps.master import MasterShellCommand
177
178 # add builder utility
179
180 c['builders'] = []
181 buildernames = []
182
183
184 def add_builder(c, name, libdir, factory, branch='',
185                 rsync=False, hour=3, minute=0):
186     workernames = []
187
188     for slave in master_private.slaves:
189         if name in slave['builders']:
190             workernames.append(slave['name'])
191
192     if workernames:
193         f = factory(name, libdir, branch, rsync)
194         c['builders'].append(BuilderConfig(name=name,
195                                            workernames=workernames,
196                                            factory=f,
197                                            tags=['blender']))
198         buildernames.append(name)
199
200         schedule_nightly_build(name, hour, minute)
201         schedule_force_build(name)
202
203
204 # common steps
205
206 def git_submodule_step(submodule):
207     return steps.Git(name=submodule + '.git',
208                      repourl='git://git.blender.org/' + submodule + '.git',
209                      mode='incremental',
210                      codebase=submodule,
211                      workdir=submodule + '.git')
212
213
214 def git_step(branch=''):
215     if branch:
216         return steps.Git(name='blender.git',
217                          repourl='git://git.blender.org/blender.git',
218                          mode='incremental',
219                          branch=branch,
220                          codebase='blender',
221                          workdir='blender.git',
222                          submodules=True)
223     else:
224         return steps.Git(name='blender.git',
225                          repourl='git://git.blender.org/blender.git',
226                          mode='incremental',
227                          codebase='blender',
228                          workdir='blender.git',
229                          submodules=True)
230
231
232 def git_submodules_update():
233     command = ['git', 'submodule', 'update', '--remote']
234     return ShellCommand(name='Submodules Update',
235                         command=command,
236                         description='updating',
237                         descriptionDone='up to date',
238                         workdir='blender.git')
239
240
241 def lib_svn_step(dir):
242     lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
243     return steps.SVN(name='lib svn',
244                      repourl=util.Interpolate(lib_repo + dir),
245                      codebase='lib svn',
246                      mode='incremental',
247                      workdir='lib/' + dir)
248
249
250 def rsync_step(id, branch, rsync_script):
251     return ShellCommand(name='rsync',
252                         command=['python', rsync_script, id, branch],
253                         description='uploading',
254                         descriptionDone='uploaded',
255                         workdir='install')
256
257
258 # generic builder
259
260 def generic_builder(id, libdir='', branch='', rsync=False):
261     filename = 'uploaded/buildbot_upload_' + id + '.zip'
262     compile_script = '../blender.git/build_files/buildbot/slave_compile.py'
263     test_script = '../blender.git/build_files/buildbot/slave_test.py'
264     pack_script = '../blender.git/build_files/buildbot/slave_pack.py'
265     rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py'
266     unpack_script = 'master_unpack.py'
267
268     f = BuildFactory()
269     if libdir != '':
270         f.addStep(lib_svn_step(libdir))
271
272     for submodule in ('blender-translations',
273                       'blender-addons',
274                       'blender-addons-contrib',
275                       'blender-dev-tools'):
276         f.addStep(git_submodule_step(submodule))
277
278     f.addStep(git_step(branch))
279     f.addStep(git_submodules_update())
280
281     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
282     f.addStep(Test(command=['python', test_script, id]))
283     f.addStep(ShellCommand(
284         name='package',
285         command=['python',
286                  pack_script,
287                  id,
288                  branch or Interpolate('%(src:blender:branch)s')],
289         description='packaging',
290         descriptionDone='packaged'))
291     if rsync:
292         f.addStep(rsync_step(id, branch, rsync_script))
293     else:
294         f.addStep(FileUpload(name='upload',
295                              workersrc='buildbot_upload.zip',
296                              masterdest=filename,
297                              maxsize=180 * 1024 * 1024,
298                              workdir='install'))
299     f.addStep(MasterShellCommand(name='unpack',
300                                  command=['python2.7',
301                                           unpack_script,
302                                           filename],
303                                  description='unpacking',
304                                  descriptionDone='unpacked'))
305     return f
306
307
308 # Builders
309
310 add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
311 add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2)
312 add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1)
313 add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1)
314 add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2)
315 add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3)
316 add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4)
317
318 # HORIZONS
319 from datetime import timedelta
320
321 c['changeHorizon'] = 300
322
323 # Configure a janitor which will delete all logs older than one month,
324 # and will run on sundays at noon.
325 c['configurators'] = [util.JanitorConfigurator(
326     logHorizon=timedelta(weeks=4),
327     hour=12,
328     dayOfWeek=6)]
329
330 # WWW
331 c['www'] = dict(port=8010,
332                 plugins={'console_view': {},
333                          'grid_view': {},
334                          'waterfall_view': {}})
335
336 # PROJECT IDENTITY
337 c['projectName'] = "Blender"
338 c['projectURL'] = "https://www.blender.org"
339
340 # Buildbot information
341 c['buildbotURL'] = "https://builder.blender.org/"
342 c['buildbotNetUsageData'] = 'basic'
343
344 # Various
345 c['db_url'] = "sqlite:///state.sqlite"
346
347 c['title'] = "Blender"
348 c['titleURL'] = "https://builder.blender.org/"
349
350 # Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
351 c['buildbotNetUsageData'] = None