build_files/buildbot/worker_pack.py

   1 # ##### BEGIN GPL LICENSE BLOCK #####
   2 #
   3 #  This program is free software; you can redistribute it and/or
   4 #  modify it under the terms of the GNU General Public License
   5 #  as published by the Free Software Foundation; either version 2
   6 #  of the License, or (at your option) any later version.
   7 #
   8 #  This program is distributed in the hope that it will be useful,
   9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 #  GNU General Public License for more details.
  12 #
  13 #  You should have received a copy of the GNU General Public License
  14 #  along with this program; if not, write to the Free Software Foundation,
  15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16 #
  17 # ##### END GPL LICENSE BLOCK #####
  18
  19 # <pep8 compliant>
  20
  21 # Runs on buildbot worker, creating a release package using the build
  22 # system and zipping it into buildbot_upload.zip. This is then uploaded
  23 # to the master in the next buildbot step.
  24
  25 import os
  26 import sys
  27
  28 from pathlib import Path
  29
  30 import buildbot_utils
  31
  32
  33 def get_package_name(builder, platform=None):
  34     info = buildbot_utils.VersionInfo(builder)
  35
  36     package_name = 'blender-' + info.full_version
  37     if platform:
  38         package_name += '-' + platform
  39     if not (builder.branch == 'master' or builder.is_release_branch):
  40         if info.is_development_build:
  41             package_name = builder.branch + "-" + package_name
  42
  43     return package_name
  44
  45
  46 def sign_file_or_directory(path):
  47     from codesign.simple_code_signer import SimpleCodeSigner
  48     code_signer = SimpleCodeSigner()
  49     code_signer.sign_file_or_directory(Path(path))
  50
  51
  52 def create_buildbot_upload_zip(builder, package_files):
  53     import zipfile
  54
  55     buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
  56     if os.path.exists(buildbot_upload_zip):
  57         os.remove(buildbot_upload_zip)
  58
  59     try:
  60         z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
  61         for filepath, filename in package_files:
  62             print("Packaged", filename)
  63             z.write(filepath, arcname=filename)
  64         z.close()
  65     except Exception as ex:
  66         sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
  67         sys.exit(1)
  68
  69
  70 def create_tar_xz(src, dest, package_name):
  71     # One extra to remove leading os.sep when cleaning root for package_root
  72     ln = len(src) + 1
  73     flist = list()
  74
  75     # Create list of tuples containing file and archive name
  76     for root, dirs, files in os.walk(src):
  77         package_root = os.path.join(package_name, root[ln:])
  78         flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
  79
  80     import tarfile
  81
  82     # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
  83     # user compiled the package. If root then unpacks it to /usr/local/ you get
  84     # a security issue.
  85     def _fakeroot(tarinfo):
  86         tarinfo.gid = 0
  87         tarinfo.gname = "root"
  88         tarinfo.uid = 0
  89         tarinfo.uname = "root"
  90         return tarinfo
  91
  92     package = tarfile.open(dest, 'w:xz', preset=9)
  93     for entry in flist:
  94         package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
  95     package.close()
  96
  97
  98 def cleanup_files(dirpath, extension):
  99     for f in os.listdir(dirpath):
 100         filepath = os.path.join(dirpath, f)
 101         if os.path.isfile(filepath) and f.endswith(extension):
 102             os.remove(filepath)
 103
 104
 105 def pack_mac(builder):
 106     info = buildbot_utils.VersionInfo(builder)
 107
 108     os.chdir(builder.build_dir)
 109     cleanup_files(builder.build_dir, '.dmg')
 110
 111     package_name = get_package_name(builder, 'macOS')
 112     package_filename = package_name + '.dmg'
 113     package_filepath = os.path.join(builder.build_dir, package_filename)
 114
 115     release_dir = os.path.join(builder.blender_dir, 'release', 'darwin')
 116     buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot')
 117     bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py')
 118
 119     command = [bundle_script]
 120     command += ['--dmg', package_filepath]
 121     if info.is_development_build:
 122         background_image = os.path.join(release_dir, 'buildbot', 'background.tif')
 123         command += ['--background-image', background_image]
 124     if builder.codesign:
 125         command += ['--codesign']
 126     command += [builder.install_dir]
 127     buildbot_utils.call(command)
 128
 129     create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
 130
 131
 132 def pack_win(builder):
 133     info = buildbot_utils.VersionInfo(builder)
 134
 135     os.chdir(builder.build_dir)
 136     cleanup_files(builder.build_dir, '.zip')
 137
 138     # CPack will add the platform name
 139     cpack_name = get_package_name(builder, None)
 140     package_name = get_package_name(builder, 'windows' + str(builder.bits))
 141
 142     command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.']
 143     buildbot_utils.call(builder.command_prefix + command)
 144     command = ['cpack', '-G', 'ZIP']
 145     buildbot_utils.call(builder.command_prefix + command)
 146
 147     package_filename = package_name + '.zip'
 148     package_filepath = os.path.join(builder.build_dir, package_filename)
 149     package_files = [(package_filepath, package_filename)]
 150
 151     if info.version_cycle == 'release':
 152         # Installer only for final release builds, otherwise will get
 153         # 'this product is already installed' messages.
 154         command = ['cpack', '-G', 'WIX']
 155         buildbot_utils.call(builder.command_prefix + command)
 156
 157         package_filename = package_name + '.msi'
 158         package_filepath = os.path.join(builder.build_dir, package_filename)
 159         if builder.codesign:
 160             sign_file_or_directory(package_filepath)
 161
 162         package_files += [(package_filepath, package_filename)]
 163
 164     create_buildbot_upload_zip(builder, package_files)
 165
 166
 167 def pack_linux(builder):
 168     blender_executable = os.path.join(builder.install_dir, 'blender')
 169
 170     info = buildbot_utils.VersionInfo(builder)
 171
 172     # Strip all unused symbols from the binaries
 173     print("Stripping binaries...")
 174     buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
 175
 176     print("Stripping python...")
 177     py_target = os.path.join(builder.install_dir, info.short_version)
 178     buildbot_utils.call(
 179         builder.command_prefix + [
 180             'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';',
 181         ],
 182     )
 183
 184     # Construct package name
 185     platform_name = 'linux64'
 186     package_name = get_package_name(builder, platform_name)
 187     package_filename = package_name + ".tar.xz"
 188
 189     print("Creating .tar.xz archive")
 190     package_filepath = builder.install_dir + '.tar.xz'
 191     create_tar_xz(builder.install_dir, package_filepath, package_name)
 192
 193     # Create buildbot_upload.zip
 194     create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
 195
 196
 197 if __name__ == "__main__":
 198     builder = buildbot_utils.create_builder_from_arguments()
 199
 200     # Make sure install directory always exists
 201     os.makedirs(builder.install_dir, exist_ok=True)
 202
 203     if builder.platform == 'mac':
 204         pack_mac(builder)
 205     elif builder.platform == 'win':
 206         pack_win(builder)
 207     elif builder.platform == 'linux':
 208         pack_linux(builder)