Attempt to fix missing branch parameter for force building windows builds
[blender-buildbot.git] / master.cfg
index ba02381727971f39e20df009dbd5c73f9aad28cc..2f4a57248c27a762c384692f8c6ba04786a18770 100644 (file)
@@ -9,81 +9,40 @@ NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
 # List of the branches available for force build
 FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
 
-"""
-Stock Twisted directory lister doesn't provide any information about last file
-modification time, we hack the class a bit in order to have such functionaliity
-:)
-"""
-
-from buildbot.status.web.base import DirectoryLister
-
-
-def get_files_and_directories(self, directory):
-    from twisted.web.static import (getTypeAndEncoding,
-                                    formatFileSize)
-    import urllib
-    import cgi
-    import time
-    import os
-    files = []
-    dirs = []
-    for path in directory:
-        url = urllib.quote(path, "/")
-        escapedPath = cgi.escape(path)
-        lastmodified = time.ctime(os.path.getmtime(
-            os.path.join(self.path, path)))
-        if os.path.isdir(os.path.join(self.path, path)):
-            url = url + '/'
-            dirs.append({'text': escapedPath + "/", 'href': url,
-                         'size': '', 'type': '[Directory]',
-                         'encoding': '',
-                         'lastmodified': lastmodified})
-        else:
-            mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
-                                                    self.contentEncodings,
-                                                    self.defaultType)
-            try:
-                size = os.stat(os.path.join(self.path, path)).st_size
-            except OSError:
-                continue
-            files.append({
-                'text': escapedPath, "href": url,
-                'type': '[%s]' % mimetype,
-                'encoding': (encoding and '[%s]' % encoding or ''),
-                'size': formatFileSize(size),
-                'lastmodified': lastmodified})
-    return dirs, files
-DirectoryLister._getFilesAndDirectories = get_files_and_directories
+# List of branches availble for addons branch selection.
+ADDONS_BRANCHES = ["master", "blender2.8"]
+
+# List of branches availble for libraries branch selection.
+LIB_BRANCHES = ["trunk"]
 
 # Dictionary that the buildmaster pays attention to.
 c = BuildmasterConfig = {}
 
-# BUILD SLAVES
+# BUILD WORKERS
 #
 # We load the slaves and their passwords from a separator file, so we can have
 # this one in SVN.
 
-from buildbot.buildslave import BuildSlave
+from buildbot.worker import Worker
 import master_private
 
-c['slaves'] = []
-
+c['workers'] = []
 for slave in master_private.slaves:
-    c['slaves'].append(BuildSlave(slave['name'], slave['password']))
+    c['workers'].append(Worker(slave['name'], slave['password']))
 
 # TCP port through which slaves connect
-
-c['slavePortnum'] = 9989
-
+c['protocols'] = {
+    "pb": {
+        "port": "tcp:{}".format(9989)
+    }
+}
 # CHANGE SOURCES
 
 from buildbot.changes.svnpoller import SVNPoller
 from buildbot.changes.gitpoller import GitPoller
 
-c['change_source'] = GitPoller(
-       'git://git.blender.org/blender.git',
-       pollinterval=1200)
-
+c['change_source'] = GitPoller('git://git.blender.org/blender.git',
+                               pollinterval=1200)
 
 # CODEBASES
 #
@@ -102,6 +61,7 @@ all_repositories = {
 def codebaseGenerator(chdict):
     return all_repositories[chdict['repository']]
 
+
 c['codebaseGenerator'] = codebaseGenerator
 
 
@@ -109,55 +69,107 @@ c['codebaseGenerator'] = codebaseGenerator
 #
 # Decide how to react to incoming changes.
 
-# from buildbot.scheduler import Scheduler
 from buildbot.schedulers import timed, forcesched
 
 c['schedulers'] = []
 
 
 def schedule_force_build(name):
-    c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
+    """
+    Makes it possible to have "Force Build" for the given builder.
+    Makes sure only reasonabel subset of properties are exposed.
+    """
+    c['schedulers'].append(forcesched.ForceScheduler(
+        name='force_' + name,
+        buttonName="Force Build",
         builderNames=[name],
         codebases=[forcesched.CodebaseParameter(
                 codebase="blender",
                 branch=forcesched.ChoiceStringParameter(
-                    name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
+                    name="branch",
+                    choices=FORCE_SCHEDULE_BRANCHES,
+                    default="master"),
                 # Do not hide revision, can be handy!
-                repository=forcesched.FixedParameter(name="repository", default="", hide=True),
-                project=forcesched.FixedParameter(name="project", default="", hide=True)),
-            # For now, hide other codebases.
-            forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
-            forcesched.CodebaseParameter(
-                codebase="blender-addons",
-                branch=forcesched.ChoiceStringParameter(
-                    name="branch", choices=["master", "blender2.8"], default="master"),
-                repository=forcesched.FixedParameter(name="repository", default="", hide=True),
-                project=forcesched.FixedParameter(name="project", default="", hide=True),
-                revision=forcesched.FixedParameter(name="revision", default="", hide=True),
-            ),
-            forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
-            forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
-            forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
+                repository=forcesched.FixedParameter(
+                    name="repository",
+                    default="",
+                    hide=True),
+                project=forcesched.FixedParameter(
+                    name="project",
+                    default="",
+                    hide=True)),
+                   # For now, hide other codebases.
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-translations"),
+                   forcesched.CodebaseParameter(
+                       codebase="blender-addons",
+                       branch=forcesched.ChoiceStringParameter(
+                           name="branch",
+                           choices=ADDONS_BRANCHES,
+                           default="master"),
+                       repository=forcesched.FixedParameter(name="repository",
+                                                            default="",
+                                                            hide=True),
+                       project=forcesched.FixedParameter(name="project",
+                                                         default="",
+                                                         hide=True),
+                       revision=forcesched.FixedParameter(name="revision",
+                                                          default="",
+                                                          hide=True),
+                   ),
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-addons-contrib"),
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-dev-tools"),
+                   forcesched.CodebaseParameter(
+                       codebase="lib svn",
+                       branch=forcesched.ChoiceStringParameter(
+                           name="branch",
+                           choices=LIB_BRANCHES,
+                           default="trunk"),
+                       repository=forcesched.FixedParameter(name="repository",
+                                                            default="",
+                                                            hide=True),
+                       project=forcesched.FixedParameter(name="project",
+                                                         default="",
+                                                         hide=True),
+                       revision=forcesched.FixedParameter(name="revision",
+                                                          default="",
+                                                          hide=True),
+                   ),
+        ],
         properties=[]))
 
 
-def schedule_build(name, hour, minute=0):
+def schedule_nightly_build(name, hour, minute=0):
+    """
+    Creates scheduler for nightly builds for a given builder.
+    """
     for current_branch in NIGHT_SCHEDULE_BRANCHES:
-        scheduler_name = "nightly " + name
+        scheduler_name = "nightly_" + name
         if current_branch:
             scheduler_name += ' ' + current_branch
         # Use special addons submodule branch when building blender2.8 branch.
         addons_branch = "master"
         if current_branch == "blender2.8":
             addons_branch = "blender2.8"
-        c['schedulers'].append(timed.Nightly(name=scheduler_name,
+        c['schedulers'].append(timed.Nightly(
+            name=scheduler_name,
             codebases={
                 "blender": {"repository": ""},
-                "blender-translations": {"repository": "", "branch": "master"},
-                "blender-addons": {"repository": "", "branch": addons_branch},
-                "blender-addons-contrib": {"repository": "", "branch": "master"},
-                "blender-dev-tools": {"repository": "", "branch": "master"},
-                "lib svn": {"repository": "", "branch": "trunk"}},
+                "blender-translations": {"repository": "",
+                                         "branch": "master"},
+                "blender-addons": {"repository": "",
+                                   "branch": addons_branch},
+                "blender-addons-contrib": {"repository": "",
+                                           "branch": "master"},
+                "blender-dev-tools": {"repository": "",
+                                      "branch": "master"},
+                "lib svn": {"repository": "",
+                            "branch": "trunk"}},
             branch=current_branch,
             builderNames=[name],
             hour=hour,
@@ -170,16 +182,15 @@ def schedule_build(name, hour, minute=0):
 # perform a build: what steps, and which slaves can execute them.
 # Note that any particular build will only take place on one slave.
 
+from buildbot.config import BuilderConfig
+from buildbot.plugins import steps, util
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate
-from buildbot.steps.source import SVN
-from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.shell import Compile
 from buildbot.steps.shell import Test
 from buildbot.steps.transfer import FileUpload
 from buildbot.steps.master import MasterShellCommand
-from buildbot.config import BuilderConfig
 
 # add builder utility
 
@@ -189,50 +200,50 @@ buildernames = []
 
 def add_builder(c, name, libdir, factory, branch='',
                 rsync=False, hour=3, minute=0):
-    slavenames = []
+    workernames = []
 
     for slave in master_private.slaves:
         if name in slave['builders']:
-            slavenames.append(slave['name'])
+            workernames.append(slave['name'])
 
-    if len(slavenames) > 0:
+    if workernames:
         f = factory(name, libdir, branch, rsync)
         c['builders'].append(BuilderConfig(name=name,
-                                           slavenames=slavenames,
+                                           workernames=workernames,
                                            factory=f,
-                                           category='blender'))
+                                           tags=['blender']))
         buildernames.append(name)
 
-        schedule_build(name, hour, minute)
+        schedule_nightly_build(name, hour, minute)
         schedule_force_build(name)
 
-# common steps
 
+# common steps
 
 def git_submodule_step(submodule):
-    return Git(name=submodule + '.git',
-               repourl='git://git.blender.org/' + submodule + '.git',
-               mode='update',
-               codebase=submodule,
-               workdir=submodule + '.git')
+    return steps.Git(name=submodule + '.git',
+                     repourl='git://git.blender.org/' + submodule + '.git',
+                     mode='incremental',
+                     codebase=submodule,
+                     workdir=submodule + '.git')
 
 
 def git_step(branch=''):
     if branch:
-        return Git(name='blender.git',
-                   repourl='git://git.blender.org/blender.git',
-                   mode='update',
-                   branch=branch,
-                   codebase='blender',
-                   workdir='blender.git',
-                   submodules=True)
+        return steps.Git(name='blender.git',
+                         repourl='git://git.blender.org/blender.git',
+                         mode='incremental',
+                         branch=branch,
+                         codebase='blender',
+                         workdir='blender.git',
+                         submodules=True)
     else:
-        return Git(name='blender.git',
-                   repourl='git://git.blender.org/blender.git',
-                   mode='update',
-                   codebase='blender',
-                   workdir='blender.git',
-                   submodules=True)
+        return steps.Git(name='blender.git',
+                         repourl='git://git.blender.org/blender.git',
+                         mode='incremental',
+                         codebase='blender',
+                         workdir='blender.git',
+                         submodules=True)
 
 
 def git_submodules_update():
@@ -245,15 +256,12 @@ def git_submodules_update():
 
 
 def lib_svn_step(dir):
-    name = "lib svn"
-    if dir == "darwin":
-        name = "C++11 lib svn"
-    return SVN(name=name,
-               baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
-               codebase='lib svn',
-               mode='update',
-               defaultBranch='trunk',
-               workdir='lib/' + dir)
+    lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
+    return steps.SVN(name='lib svn',
+                     repourl=util.Interpolate(lib_repo + dir),
+                     codebase='lib svn',
+                     mode='incremental',
+                     workdir='lib/' + dir)
 
 
 def rsync_step(id, branch, rsync_script):
@@ -263,8 +271,8 @@ def rsync_step(id, branch, rsync_script):
                         descriptionDone='uploaded',
                         workdir='install')
 
-# generic builder
 
+# generic builder
 
 def generic_builder(id, libdir='', branch='', rsync=False):
     filename = 'uploaded/buildbot_upload_' + id + '.zip'
@@ -277,9 +285,6 @@ def generic_builder(id, libdir='', branch='', rsync=False):
     f = BuildFactory()
     if libdir != '':
         f.addStep(lib_svn_step(libdir))
-        # Special trick to make sure we always have all the libs.
-        if libdir.startswith("darwin"):
-            f.addStep(lib_svn_step("darwin"))
 
     for submodule in ('blender-translations',
                       'blender-addons',
@@ -292,85 +297,72 @@ def generic_builder(id, libdir='', branch='', rsync=False):
 
     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
     f.addStep(Test(command=['python', test_script, id]))
-    f.addStep(ShellCommand(name='package',
-                           command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
-                           description='packaging',
-                           descriptionDone='packaged'))
+    f.addStep(ShellCommand(
+        name='package',
+        command=['python',
+                 pack_script,
+                 id,
+                 branch or Interpolate('%(src:blender:branch)s')],
+        description='packaging',
+        descriptionDone='packaged'))
     if rsync:
         f.addStep(rsync_step(id, branch, rsync_script))
     else:
         f.addStep(FileUpload(name='upload',
-                             slavesrc='buildbot_upload.zip',
+                             workersrc='buildbot_upload.zip',
                              masterdest=filename,
-                             maxsize=150 * 1024 * 1024,
+                             maxsize=180 * 1024 * 1024,
                              workdir='install'))
     f.addStep(MasterShellCommand(name='unpack',
-                                 command=['python2.7', unpack_script, filename],
+                                 command=['python2.7',
+                                          unpack_script,
+                                          filename],
                                  description='unpacking',
                                  descriptionDone='unpacked'))
     return f
 
+
 # Builders
 
-add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
-# add_builder(c, 'linux_glibc211_i686_cmake', '', generic_builder, hour=1)
-# add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
-add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=3)
-add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=4)
-add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
-add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
-add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=5)
-add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=6)
-
-# STATUS TARGETS
-#
-# 'status' is a list of Status Targets. The results of each build will be
-# pushed to these targets. buildbot/status/*.py has a variety to choose from,
-# including web pages, email senders, and IRC bots.
+add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
+add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2)
+add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1)
+add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1)
+add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2)
+add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3)
+add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4)
 
-c['status'] = []
+# HORIZONS
+from datetime import timedelta
 
-from buildbot.status import html
-from buildbot.status.web import authz
-from buildbot.status.web import auth
+c['changeHorizon'] = 300
 
-users = []
-for slave in master_private.slaves:
-    users += [(slave['name'], slave['password'])]
-
-authz_cfg = authz.Authz(
-    auth=auth.BasicAuth(users),
-    # change any of these to True to enable; see the manual for more
-    # options
-    gracefulShutdown=False,
-    forceBuild=True,  # use this to test your slave once it is set up
-    forceAllBuilds=False,
-    pingBuilder=False,
-    stopBuild=True,
-    stopAllBuilds=False,
-    cancelPendingBuild=True,
-)
-
-c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-#c['status'].append(html.WebStatus(http_port=8010))
+# Configure a janitor which will delete all logs older than one month,
+# and will run on sundays at noon.
+c['configurators'] = [util.JanitorConfigurator(
+    logHorizon=timedelta(weeks=4),
+    hour=12,
+    dayOfWeek=6)]
 
-# PROJECT IDENTITY
+# WWW
+c['www'] = dict(port=8010,
+                plugins={'console_view': {},
+                         'grid_view': {},
+                         'waterfall_view': {}})
 
+# PROJECT IDENTITY
 c['projectName'] = "Blender"
-c['projectURL'] = "http://www.blender.org"
+c['projectURL'] = "https://www.blender.org"
 
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server (usually the html.WebStatus page) is visible. This
-# typically uses the port number set in the Waterfall 'status' entry, but
-# with an externally-visible host name which the buildbot cannot figure out
-# without some help.
+# Buildbot information
+c['buildbotURL'] = "https://builder.blender.org/admin/"
+c['buildbotNetUsageData'] = 'basic'
 
-c['buildbotURL'] = "http://builder.blender.org/"
+# Various
+c['db_url'] = "sqlite:///state.sqlite"
 
-# DB URL
-#
-# This specifies what database buildbot uses to store change and scheduler
-# state.  You can leave this at its default for all but the largest
-# installations.
+c['title'] = "Blender"
+c['titleURL'] = "https://builder.blender.org/"
 
-c['db_url'] = "sqlite:///state.sqlite"
+# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
+c['buildbotNetUsageData'] = None