Attempt to fix missing branch parameter for force building windows builds
[blender-buildbot.git] / master.cfg
index fb4219b583d6006b211bf2433259c5bb96d89251..2f4a57248c27a762c384692f8c6ba04786a18770 100644 (file)
@@ -4,86 +4,45 @@
 # <pep8 compliant>
 
 # List of the branches being built automatically overnight
-NIGHT_SCHEDULE_BRANCHES = [None]
+NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"]
 
 # List of the branches available for force build
-FORCE_SCHEDULE_BRANCHES = ["master", "gooseberry", "experimental-build"]
-
-"""
-Stock Twisted directory lister doesn't provide any information about last file
-modification time, we hack the class a bit in order to have such functionaliity
-:)
-"""
-
-from buildbot.status.web.base import DirectoryLister
-
-
-def get_files_and_directories(self, directory):
-    from twisted.web.static import (getTypeAndEncoding,
-                                    formatFileSize)
-    import urllib
-    import cgi
-    import time
-    import os
-    files = []
-    dirs = []
-    for path in directory:
-        url = urllib.quote(path, "/")
-        escapedPath = cgi.escape(path)
-        lastmodified = time.ctime(os.path.getmtime(
-            os.path.join(self.path, path)))
-        if os.path.isdir(os.path.join(self.path, path)):
-            url = url + '/'
-            dirs.append({'text': escapedPath + "/", 'href': url,
-                         'size': '', 'type': '[Directory]',
-                         'encoding': '',
-                         'lastmodified': lastmodified})
-        else:
-            mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
-                                                    self.contentEncodings,
-                                                    self.defaultType)
-            try:
-                size = os.stat(os.path.join(self.path, path)).st_size
-            except OSError:
-                continue
-            files.append({
-                'text': escapedPath, "href": url,
-                'type': '[%s]' % mimetype,
-                'encoding': (encoding and '[%s]' % encoding or ''),
-                'size': formatFileSize(size),
-                'lastmodified': lastmodified})
-    return dirs, files
-DirectoryLister._getFilesAndDirectories = get_files_and_directories
+FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"]
+
+# List of branches availble for addons branch selection.
+ADDONS_BRANCHES = ["master", "blender2.8"]
+
+# List of branches availble for libraries branch selection.
+LIB_BRANCHES = ["trunk"]
 
 # Dictionary that the buildmaster pays attention to.
 c = BuildmasterConfig = {}
 
-# BUILD SLAVES
+# BUILD WORKERS
 #
 # We load the slaves and their passwords from a separator file, so we can have
 # this one in SVN.
 
-from buildbot.buildslave import BuildSlave
+from buildbot.worker import Worker
 import master_private
 
-c['slaves'] = []
-
+c['workers'] = []
 for slave in master_private.slaves:
-    c['slaves'].append(BuildSlave(slave['name'], slave['password']))
+    c['workers'].append(Worker(slave['name'], slave['password']))
 
 # TCP port through which slaves connect
-
-c['slavePortnum'] = 9989
-
+c['protocols'] = {
+    "pb": {
+        "port": "tcp:{}".format(9989)
+    }
+}
 # CHANGE SOURCES
 
 from buildbot.changes.svnpoller import SVNPoller
 from buildbot.changes.gitpoller import GitPoller
 
-c['change_source'] = GitPoller(
-       'git://git.blender.org/blender.git',
-       pollinterval=1200)
-
+c['change_source'] = GitPoller('git://git.blender.org/blender.git',
+                               pollinterval=1200)
 
 # CODEBASES
 #
@@ -102,6 +61,7 @@ all_repositories = {
 def codebaseGenerator(chdict):
     return all_repositories[chdict['repository']]
 
+
 c['codebaseGenerator'] = codebaseGenerator
 
 
@@ -109,44 +69,107 @@ c['codebaseGenerator'] = codebaseGenerator
 #
 # Decide how to react to incoming changes.
 
-# from buildbot.scheduler import Scheduler
 from buildbot.schedulers import timed, forcesched
 
 c['schedulers'] = []
 
 
 def schedule_force_build(name):
-    c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name,
+    """
+    Makes it possible to have "Force Build" for the given builder.
+    Makes sure only reasonabel subset of properties are exposed.
+    """
+    c['schedulers'].append(forcesched.ForceScheduler(
+        name='force_' + name,
+        buttonName="Force Build",
         builderNames=[name],
         codebases=[forcesched.CodebaseParameter(
                 codebase="blender",
                 branch=forcesched.ChoiceStringParameter(
-                    name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"),
+                    name="branch",
+                    choices=FORCE_SCHEDULE_BRANCHES,
+                    default="master"),
                 # Do not hide revision, can be handy!
-                repository=forcesched.FixedParameter(name="repository", default="", hide=True),
-                project=forcesched.FixedParameter(name="project", default="", hide=True)),
-            # For now, hide other codebases.
-            forcesched.CodebaseParameter(hide=True, codebase="blender-translations"),
-            forcesched.CodebaseParameter(hide=True, codebase="blender-addons"),
-            forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"),
-            forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"),
-            forcesched.CodebaseParameter(hide=True, codebase="lib svn")],
+                repository=forcesched.FixedParameter(
+                    name="repository",
+                    default="",
+                    hide=True),
+                project=forcesched.FixedParameter(
+                    name="project",
+                    default="",
+                    hide=True)),
+                   # For now, hide other codebases.
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-translations"),
+                   forcesched.CodebaseParameter(
+                       codebase="blender-addons",
+                       branch=forcesched.ChoiceStringParameter(
+                           name="branch",
+                           choices=ADDONS_BRANCHES,
+                           default="master"),
+                       repository=forcesched.FixedParameter(name="repository",
+                                                            default="",
+                                                            hide=True),
+                       project=forcesched.FixedParameter(name="project",
+                                                         default="",
+                                                         hide=True),
+                       revision=forcesched.FixedParameter(name="revision",
+                                                          default="",
+                                                          hide=True),
+                   ),
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-addons-contrib"),
+                   forcesched.CodebaseParameter(
+                       hide=True,
+                       codebase="blender-dev-tools"),
+                   forcesched.CodebaseParameter(
+                       codebase="lib svn",
+                       branch=forcesched.ChoiceStringParameter(
+                           name="branch",
+                           choices=LIB_BRANCHES,
+                           default="trunk"),
+                       repository=forcesched.FixedParameter(name="repository",
+                                                            default="",
+                                                            hide=True),
+                       project=forcesched.FixedParameter(name="project",
+                                                         default="",
+                                                         hide=True),
+                       revision=forcesched.FixedParameter(name="revision",
+                                                          default="",
+                                                          hide=True),
+                   ),
+        ],
         properties=[]))
 
 
-def schedule_build(name, hour, minute=0):
+def schedule_nightly_build(name, hour, minute=0):
+    """
+    Creates scheduler for nightly builds for a given builder.
+    """
     for current_branch in NIGHT_SCHEDULE_BRANCHES:
-        scheduler_name = "nightly " + name
+        scheduler_name = "nightly_" + name
         if current_branch:
             scheduler_name += ' ' + current_branch
-        c['schedulers'].append(timed.Nightly(name=scheduler_name,
+        # Use special addons submodule branch when building blender2.8 branch.
+        addons_branch = "master"
+        if current_branch == "blender2.8":
+            addons_branch = "blender2.8"
+        c['schedulers'].append(timed.Nightly(
+            name=scheduler_name,
             codebases={
                 "blender": {"repository": ""},
-                "blender-translations": {"repository": "", "branch": "master"},
-                "blender-addons": {"repository": "", "branch": "master"},
-                "blender-addons-contrib": {"repository": "", "branch": "master"},
-                "blender-dev-tools": {"repository": "", "branch": "master"},
-                "lib svn": {"repository": "", "branch": "trunk"}},
+                "blender-translations": {"repository": "",
+                                         "branch": "master"},
+                "blender-addons": {"repository": "",
+                                   "branch": addons_branch},
+                "blender-addons-contrib": {"repository": "",
+                                           "branch": "master"},
+                "blender-dev-tools": {"repository": "",
+                                      "branch": "master"},
+                "lib svn": {"repository": "",
+                            "branch": "trunk"}},
             branch=current_branch,
             builderNames=[name],
             hour=hour,
@@ -159,16 +182,15 @@ def schedule_build(name, hour, minute=0):
 # perform a build: what steps, and which slaves can execute them.
 # Note that any particular build will only take place on one slave.
 
+from buildbot.config import BuilderConfig
+from buildbot.plugins import steps, util
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate
-from buildbot.steps.source import SVN
-from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.shell import Compile
 from buildbot.steps.shell import Test
 from buildbot.steps.transfer import FileUpload
 from buildbot.steps.master import MasterShellCommand
-from buildbot.config import BuilderConfig
 
 # add builder utility
 
@@ -178,55 +200,54 @@ buildernames = []
 
 def add_builder(c, name, libdir, factory, branch='',
                 rsync=False, hour=3, minute=0):
-    slavenames = []
+    workernames = []
 
     for slave in master_private.slaves:
         if name in slave['builders']:
-            slavenames.append(slave['name'])
+            workernames.append(slave['name'])
 
-    if len(slavenames) > 0:
+    if workernames:
         f = factory(name, libdir, branch, rsync)
         c['builders'].append(BuilderConfig(name=name,
-                                           slavenames=slavenames,
+                                           workernames=workernames,
                                            factory=f,
-                                           category='blender'))
+                                           tags=['blender']))
         buildernames.append(name)
 
-        schedule_build(name, hour, minute)
+        schedule_nightly_build(name, hour, minute)
         schedule_force_build(name)
 
-# common steps
 
+# common steps
 
 def git_submodule_step(submodule):
-    return Git(name=submodule + '.git',
-               repourl='git://git.blender.org/' + submodule + '.git',
-               mode='update',
-               codebase=submodule,
-               workdir=submodule + '.git')
+    return steps.Git(name=submodule + '.git',
+                     repourl='git://git.blender.org/' + submodule + '.git',
+                     mode='incremental',
+                     codebase=submodule,
+                     workdir=submodule + '.git')
 
 
 def git_step(branch=''):
     if branch:
-        return Git(name='blender.git',
-                   repourl='git://git.blender.org/blender.git',
-                   mode='update',
-                   branch=branch,
-                   codebase='blender',
-                   workdir='blender.git',
-                   submodules=True)
+        return steps.Git(name='blender.git',
+                         repourl='git://git.blender.org/blender.git',
+                         mode='incremental',
+                         branch=branch,
+                         codebase='blender',
+                         workdir='blender.git',
+                         submodules=True)
     else:
-        return Git(name='blender.git',
-                   repourl='git://git.blender.org/blender.git',
-                   mode='update',
-                   codebase='blender',
-                   workdir='blender.git',
-                   submodules=True)
+        return steps.Git(name='blender.git',
+                         repourl='git://git.blender.org/blender.git',
+                         mode='incremental',
+                         codebase='blender',
+                         workdir='blender.git',
+                         submodules=True)
 
 
 def git_submodules_update():
-    command = ['git', 'submodule', 'foreach', '--recursive',
-               'git', 'pull', 'origin', 'master']
+    command = ['git', 'submodule', 'update', '--remote']
     return ShellCommand(name='Submodules Update',
                         command=command,
                         description='updating',
@@ -235,12 +256,12 @@ def git_submodules_update():
 
 
 def lib_svn_step(dir):
-    return SVN(name='lib svn',
-               baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir,
-               codebase='lib svn',
-               mode='update',
-               defaultBranch='trunk',
-               workdir='lib/' + dir)
+    lib_repo = 'https://svn.blender.org/svnroot/bf-blender/%(src::branch)s/lib/'
+    return steps.SVN(name='lib svn',
+                     repourl=util.Interpolate(lib_repo + dir),
+                     codebase='lib svn',
+                     mode='incremental',
+                     workdir='lib/' + dir)
 
 
 def rsync_step(id, branch, rsync_script):
@@ -250,8 +271,8 @@ def rsync_step(id, branch, rsync_script):
                         descriptionDone='uploaded',
                         workdir='install')
 
-# generic builder
 
+# generic builder
 
 def generic_builder(id, libdir='', branch='', rsync=False):
     filename = 'uploaded/buildbot_upload_' + id + '.zip'
@@ -276,84 +297,72 @@ def generic_builder(id, libdir='', branch='', rsync=False):
 
     f.addStep(Compile(command=['python', compile_script, id], timeout=3600))
     f.addStep(Test(command=['python', test_script, id]))
-    f.addStep(ShellCommand(name='package',
-                           command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')],
-                           description='packaging',
-                           descriptionDone='packaged'))
+    f.addStep(ShellCommand(
+        name='package',
+        command=['python',
+                 pack_script,
+                 id,
+                 branch or Interpolate('%(src:blender:branch)s')],
+        description='packaging',
+        descriptionDone='packaged'))
     if rsync:
         f.addStep(rsync_step(id, branch, rsync_script))
     else:
         f.addStep(FileUpload(name='upload',
-                             slavesrc='buildbot_upload.zip',
+                             workersrc='buildbot_upload.zip',
                              masterdest=filename,
-                             maxsize=150 * 1024 * 1024,
+                             maxsize=180 * 1024 * 1024,
                              workdir='install'))
     f.addStep(MasterShellCommand(name='unpack',
-                                 command=['python2.7', unpack_script, filename],
+                                 command=['python2.7',
+                                          unpack_script,
+                                          filename],
                                  description='unpacking',
                                  descriptionDone='unpacked'))
     return f
 
+
 # Builders
 
-add_builder(c, 'mac_x86_64_10_6_cmake', 'darwin-9.x.universal', generic_builder, hour=5)
-add_builder(c, 'linux_glibc211_i686_cmake', '', generic_builder, hour=1)
-add_builder(c, 'linux_glibc211_x86_64_cmake', '', generic_builder, hour=2)
-add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=3)
-add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=4)
-add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=3)
-add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=4)
-add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=5)
+add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1)
+add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2)
+add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1)
+add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1)
+add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2)
+add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3)
+add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4)
 
-# STATUS TARGETS
-#
-# 'status' is a list of Status Targets. The results of each build will be
-# pushed to these targets. buildbot/status/*.py has a variety to choose from,
-# including web pages, email senders, and IRC bots.
+# HORIZONS
+from datetime import timedelta
 
-c['status'] = []
+c['changeHorizon'] = 300
 
-from buildbot.status import html
-from buildbot.status.web import authz
-from buildbot.status.web import auth
+# Configure a janitor which will delete all logs older than one month,
+# and will run on sundays at noon.
+c['configurators'] = [util.JanitorConfigurator(
+    logHorizon=timedelta(weeks=4),
+    hour=12,
+    dayOfWeek=6)]
 
-users = []
-for slave in master_private.slaves:
-    users += [(slave['name'], slave['password'])]
-
-authz_cfg = authz.Authz(
-    auth=auth.BasicAuth(users),
-    # change any of these to True to enable; see the manual for more
-    # options
-    gracefulShutdown=False,
-    forceBuild=True,  # use this to test your slave once it is set up
-    forceAllBuilds=False,
-    pingBuilder=False,
-    stopBuild=True,
-    stopAllBuilds=False,
-    cancelPendingBuild=True,
-)
-
-c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-#c['status'].append(html.WebStatus(http_port=8010))
+# WWW
+c['www'] = dict(port=8010,
+                plugins={'console_view': {},
+                         'grid_view': {},
+                         'waterfall_view': {}})
 
 # PROJECT IDENTITY
-
 c['projectName'] = "Blender"
-c['projectURL'] = "http://www.blender.org"
+c['projectURL'] = "https://www.blender.org"
 
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server (usually the html.WebStatus page) is visible. This
-# typically uses the port number set in the Waterfall 'status' entry, but
-# with an externally-visible host name which the buildbot cannot figure out
-# without some help.
+# Buildbot information
+c['buildbotURL'] = "https://builder.blender.org/admin/"
+c['buildbotNetUsageData'] = 'basic'
 
-c['buildbotURL'] = "http://builder.blender.org/"
+# Various
+c['db_url'] = "sqlite:///state.sqlite"
 
-# DB URL
-#
-# This specifies what database buildbot uses to store change and scheduler
-# state.  You can leave this at its default for all but the largest
-# installations.
+c['title'] = "Blender"
+c['titleURL'] = "https://builder.blender.org/"
 
-c['db_url'] = "sqlite:///state.sqlite"
+# Disable sending of 'buildbotNetUsageData' for now, to improve startup time.
+c['buildbotNetUsageData'] = None