build_files/buildbot/worker_bundle_dmg.py

   1 #!/usr/bin/env python3
   2
   3 # ##### BEGIN GPL LICENSE BLOCK #####
   4 #
   5 #  This program is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU General Public License
   7 #  as published by the Free Software Foundation; either version 2
   8 #  of the License, or (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software Foundation,
  17 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18 #
  19 # ##### END GPL LICENSE BLOCK #####
  20
  21 import argparse
  22 import re
  23 import shutil
  24 import subprocess
  25 import sys
  26 import time
  27
  28 from pathlib import Path
  29 from tempfile import TemporaryDirectory, NamedTemporaryFile
  30 from typing import List
  31
  32 BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
  33 CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py'
  34 BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
  35 DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
  36
  37
  38 # Extra size which is added on top of actual files size when estimating size
  39 # of destination DNG.
  40 EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
  41
  42 ################################################################################
  43 # Common utilities
  44
  45
  46 def get_directory_size(root_directory: Path) -> int:
  47     """
  48     Get size of directory on disk
  49     """
  50
  51     total_size = 0
  52     for file in root_directory.glob('**/*'):
  53         total_size += file.lstat().st_size
  54     return total_size
  55
  56
  57 ################################################################################
  58 # DMG bundling specific logic
  59
  60 def create_argument_parser():
  61     parser = argparse.ArgumentParser()
  62     parser.add_argument(
  63         'source_dir',
  64         type=Path,
  65         help='Source directory which points to either existing .app bundle'
  66              'or to a directory with .app bundles.')
  67     parser.add_argument(
  68         '--background-image',
  69         type=Path,
  70         help="Optional background picture which will be set on the DMG."
  71              "If not provided default Blender's one is used.")
  72     parser.add_argument(
  73         '--volume-name',
  74         type=str,
  75         help='Optional name of a volume which will be used for DMG.')
  76     parser.add_argument(
  77         '--dmg',
  78         type=Path,
  79         help='Optional argument which points to a final DMG file name.')
  80     parser.add_argument(
  81         '--applescript',
  82         type=Path,
  83         help="Optional path to applescript to set up folder looks of DMG."
  84              "If not provided default Blender's one is used.")
  85     parser.add_argument(
  86         '--codesign',
  87         action="store_true",
  88         help="Code sign and notarize DMG contents.")
  89     return parser
  90
  91
  92 def collect_app_bundles(source_dir: Path) -> List[Path]:
  93     """
  94     Collect all app bundles which are to be put into DMG
  95
  96     If the source directory points to FOO.app it will be the only app bundle
  97     packed.
  98
  99     Otherwise all .app bundles from given directory are placed to a single
 100     DMG.
 101     """
 102
 103     if source_dir.name.endswith('.app'):
 104         return [source_dir]
 105
 106     app_bundles = []
 107     for filename in source_dir.glob('*'):
 108         if not filename.is_dir():
 109             continue
 110         if not filename.name.endswith('.app'):
 111             continue
 112
 113         app_bundles.append(filename)
 114
 115     return app_bundles
 116
 117
 118 def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
 119     app_bundles = collect_app_bundles(source_dir)
 120
 121     if not app_bundles:
 122         print('No app bundles found for packing')
 123         return
 124
 125     print(f'Found {len(app_bundles)} to pack:')
 126     for app_bundle in app_bundles:
 127         print(f'- {app_bundle}')
 128
 129     return app_bundles
 130
 131
 132 def estimate_dmg_size(app_bundles: List[Path]) -> int:
 133     """
 134     Estimate size of DMG to hold requested app bundles
 135
 136     The size is based on actual size of all files in all bundles plus some
 137     space to compensate for different size-on-disk plus some space to hold
 138     codesign signatures.
 139
 140     Is better to be on a high side since the empty space is compressed, but
 141     lack of space might cause silent failures later on.
 142     """
 143
 144     app_bundles_size = 0
 145     for app_bundle in app_bundles:
 146         app_bundles_size += get_directory_size(app_bundle)
 147
 148     return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
 149
 150
 151 def copy_app_bundles_to_directory(app_bundles: List[Path],
 152                                   directory: Path) -> None:
 153     """
 154     Copy all bundles to a given directory
 155
 156     This directory is what the DMG will be created from.
 157     """
 158     for app_bundle in app_bundles:
 159         print(f'Copying {app_bundle.name}...')
 160         shutil.copytree(app_bundle, directory / app_bundle.name)
 161
 162
 163 def get_main_app_bundle(app_bundles: List[Path]) -> Path:
 164     """
 165     Get application bundle main for the installation
 166     """
 167     return app_bundles[0]
 168
 169
 170 def create_dmg_image(app_bundles: List[Path],
 171                      dmg_filepath: Path,
 172                      volume_name: str) -> None:
 173     """
 174     Create DMG disk image and put app bundles in it
 175
 176     No DMG configuration or codesigning is happening here.
 177     """
 178
 179     if dmg_filepath.exists():
 180         print(f'Removing existing writable DMG {dmg_filepath}...')
 181         dmg_filepath.unlink()
 182
 183     print('Preparing directory with app bundles for the DMG...')
 184     with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
 185         # Copy all bundles to a clean directory.
 186         content_dir = Path(content_dir_str)
 187         copy_app_bundles_to_directory(app_bundles, content_dir)
 188
 189         # Estimate size of the DMG.
 190         dmg_size = estimate_dmg_size(app_bundles)
 191         print(f'Estimated DMG size: {dmg_size:,} bytes.')
 192
 193         # Create the DMG.
 194         print(f'Creating writable DMG {dmg_filepath}')
 195         command = ('hdiutil',
 196                    'create',
 197                    '-size', str(dmg_size),
 198                    '-fs', 'HFS+',
 199                    '-srcfolder', content_dir,
 200                    '-volname', volume_name,
 201                    '-format', 'UDRW',
 202                    dmg_filepath)
 203         subprocess.run(command)
 204
 205
 206 def get_writable_dmg_filepath(dmg_filepath: Path):
 207     """
 208     Get file path for writable DMG image
 209     """
 210     parent = dmg_filepath.parent
 211     return parent / (dmg_filepath.stem + '-temp.dmg')
 212
 213
 214 def mount_readwrite_dmg(dmg_filepath: Path) -> None:
 215     """
 216     Mount writable DMG
 217
 218     Mounting point would be /Volumes/<volume name>
 219     """
 220
 221     print(f'Mounting read-write DMG ${dmg_filepath}')
 222     command = ('hdiutil',
 223                'attach', '-readwrite',
 224                '-noverify',
 225                '-noautoopen',
 226                dmg_filepath)
 227     subprocess.run(command)
 228
 229
 230 def get_mount_directory_for_volume_name(volume_name: str) -> Path:
 231     """
 232     Get directory under which the volume will be mounted
 233     """
 234
 235     return Path('/Volumes') / volume_name
 236
 237
 238 def eject_volume(volume_name: str) -> None:
 239     """
 240     Eject given volume, if mounted
 241     """
 242     mount_directory = get_mount_directory_for_volume_name(volume_name)
 243     if not mount_directory.exists():
 244         return
 245     mount_directory_str = str(mount_directory)
 246
 247     print(f'Ejecting volume {volume_name}')
 248
 249     # Figure out which device to eject.
 250     mount_output = subprocess.check_output(['mount']).decode()
 251     device = ''
 252     for line in mount_output.splitlines():
 253         if f'on {mount_directory_str} (' not in line:
 254             continue
 255         tokens = line.split(' ', 3)
 256         if len(tokens) < 3:
 257             continue
 258         if tokens[1] != 'on':
 259             continue
 260         if device:
 261             raise Exception(
 262                 f'Multiple devices found for mounting point {mount_directory}')
 263         device = tokens[0]
 264
 265     if not device:
 266         raise Exception(
 267             f'No device found for mounting point {mount_directory}')
 268
 269     print(f'{mount_directory} is mounted as device {device}, ejecting...')
 270     subprocess.run(['diskutil', 'eject', device])
 271
 272
 273 def copy_background_if_needed(background_image_filepath: Path,
 274                               mount_directory: Path) -> None:
 275     """
 276     Copy background to the DMG
 277
 278     If the background image is not specified it will not be copied.
 279     """
 280
 281     if not background_image_filepath:
 282         print('No background image provided.')
 283         return
 284
 285     print(f'Copying background image {background_image_filepath}')
 286
 287     destination_dir = mount_directory / '.background'
 288     destination_dir.mkdir(exist_ok=True)
 289
 290     destination_filepath = destination_dir / background_image_filepath.name
 291     shutil.copy(background_image_filepath, destination_filepath)
 292
 293
 294 def create_applications_link(mount_directory: Path) -> None:
 295     """
 296     Create link to /Applications in the given location
 297     """
 298
 299     print('Creating link to /Applications')
 300
 301     command = ('ln', '-s', '/Applications', mount_directory / ' ')
 302     subprocess.run(command)
 303
 304
 305 def run_applescript(applescript: Path,
 306                     volume_name: str,
 307                     app_bundles: List[Path],
 308                     background_image_filepath: Path) -> None:
 309     """
 310     Run given applescript to adjust look and feel of the DMG
 311     """
 312
 313     main_app_bundle = get_main_app_bundle(app_bundles)
 314
 315     with NamedTemporaryFile(
 316             mode='w', suffix='.applescript') as temp_applescript:
 317         print('Adjusting applescript for volume name...')
 318         # Adjust script to the specific volume name.
 319         with open(applescript, mode='r') as input:
 320             for line in input.readlines():
 321                 stripped_line = line.strip()
 322                 if stripped_line.startswith('tell disk'):
 323                     line = re.sub('tell disk ".*"',
 324                                   f'tell disk "{volume_name}"',
 325                                   line)
 326                 elif stripped_line.startswith('set background picture'):
 327                     if not background_image_filepath:
 328                         continue
 329                     else:
 330                         background_image_short = \
 331                             '.background:' + background_image_filepath.name
 332                         line = re.sub('to file ".*"',
 333                                       f'to file "{background_image_short}"',
 334                                       line)
 335                 line = line.replace('blender.app', main_app_bundle.name)
 336                 temp_applescript.write(line)
 337
 338         temp_applescript.flush()
 339
 340         print('Running applescript...')
 341         command = ('osascript',  temp_applescript.name)
 342         subprocess.run(command)
 343
 344         print('Waiting for applescript...')
 345
 346         # NOTE: This is copied from bundle.sh. The exact reason for sleep is
 347         # still remained a mystery.
 348         time.sleep(5)
 349
 350
 351 def codesign(subject: Path):
 352     """
 353     Codesign file or directory
 354
 355     NOTE: For DMG it will also notarize.
 356     """
 357
 358     command = (CODESIGN_SCRIPT, subject)
 359     subprocess.run(command)
 360
 361
 362 def codesign_app_bundles_in_dmg(mount_directory: str) -> None:
 363     """
 364     Code sign all binaries and bundles in the mounted directory
 365     """
 366
 367     print(f'Codesigning all app bundles in {mount_directory}')
 368     codesign(mount_directory)
 369
 370
 371 def codesign_and_notarize_dmg(dmg_filepath: Path) -> None:
 372     """
 373     Run codesign and notarization pipeline on the DMG
 374     """
 375
 376     print(f'Codesigning and notarizing DMG {dmg_filepath}')
 377     codesign(dmg_filepath)
 378
 379
 380 def compress_dmg(writable_dmg_filepath: Path,
 381                  final_dmg_filepath: Path) -> None:
 382     """
 383     Compress temporary read-write DMG
 384     """
 385     command = ('hdiutil', 'convert',
 386                writable_dmg_filepath,
 387                '-format', 'UDZO',
 388                '-o', final_dmg_filepath)
 389
 390     if final_dmg_filepath.exists():
 391         print(f'Removing old compressed DMG {final_dmg_filepath}')
 392         final_dmg_filepath.unlink()
 393
 394     print('Compressing disk image...')
 395     subprocess.run(command)
 396
 397
 398 def create_final_dmg(app_bundles: List[Path],
 399                      dmg_filepath: Path,
 400                      background_image_filepath: Path,
 401                      volume_name: str,
 402                      applescript: Path,
 403                      codesign: bool) -> None:
 404     """
 405     Create DMG with all app bundles
 406
 407     Will take care configuring background, signing all binaries and app bundles
 408     and notarizing the DMG.
 409     """
 410
 411     print('Running all routines to create final DMG')
 412
 413     writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath)
 414     mount_directory = get_mount_directory_for_volume_name(volume_name)
 415
 416     # Make sure volume is not mounted.
 417     # If it is mounted it will prevent removing old DMG files and could make
 418     # it so app bundles are copied to the wrong place.
 419     eject_volume(volume_name)
 420
 421     create_dmg_image(app_bundles, writable_dmg_filepath, volume_name)
 422
 423     mount_readwrite_dmg(writable_dmg_filepath)
 424
 425     # Run codesign first, prior to copying amything else.
 426     #
 427     # This allows to recurs into the content of bundles without worrying about
 428     # possible interfereice of Application symlink.
 429     if codesign:
 430         codesign_app_bundles_in_dmg(mount_directory)
 431
 432     copy_background_if_needed(background_image_filepath, mount_directory)
 433     create_applications_link(mount_directory)
 434     run_applescript(applescript, volume_name, app_bundles,
 435                     background_image_filepath)
 436
 437     print('Ejecting read-write DMG image...')
 438     eject_volume(volume_name)
 439
 440     compress_dmg(writable_dmg_filepath, dmg_filepath)
 441     writable_dmg_filepath.unlink()
 442
 443     if codesign:
 444         codesign_and_notarize_dmg(dmg_filepath)
 445
 446
 447 def ensure_dmg_extension(filepath: Path) -> Path:
 448     """
 449     Make sure given file have .dmg extension
 450     """
 451
 452     if filepath.suffix != '.dmg':
 453         return filepath.with_suffix(f'{filepath.suffix}.dmg')
 454     return filepath
 455
 456
 457 def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path:
 458     """
 459     Get full file path for the final DMG image
 460
 461     Will use the provided one when possible, otherwise will deduct it from
 462     app bundles.
 463
 464     If the name is deducted, the DMG is stored in the current directory.
 465     """
 466
 467     if requested_name:
 468         return ensure_dmg_extension(requested_name.absolute())
 469
 470     # TODO(sergey): This is not necessarily the main one.
 471     main_bundle = app_bundles[0]
 472     # Strip .app from the name
 473     return Path(main_bundle.name[:-4] + '.dmg').absolute()
 474
 475
 476 def get_background_image(requested_background_image: Path) -> Path:
 477     """
 478     Get effective filepath for the background image
 479     """
 480
 481     if requested_background_image:
 482         return requested_background_image.absolute()
 483
 484     return DARWIN_DIRECTORY / 'background.tif'
 485
 486
 487 def get_applescript(requested_applescript: Path) -> Path:
 488     """
 489     Get effective filepath for the applescript
 490     """
 491
 492     if requested_applescript:
 493         return requested_applescript.absolute()
 494
 495     return DARWIN_DIRECTORY / 'blender.applescript'
 496
 497
 498 def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str:
 499     """
 500     Deduct volume name from the DMG path
 501
 502     Will use first part of the DMG file name prior to dash.
 503     """
 504
 505     tokens = dmg_filepath.stem.split('-')
 506     words = tokens[0].split()
 507
 508     return ' '.join(word.capitalize() for word in words)
 509
 510
 511 def get_volume_name(requested_volume_name: str,
 512                     dmg_filepath: Path) -> str:
 513     """
 514     Get effective name for DMG volume
 515     """
 516
 517     if requested_volume_name:
 518         return requested_volume_name
 519
 520     return get_volume_name_from_dmg_filepath(dmg_filepath)
 521
 522
 523 def main():
 524     parser = create_argument_parser()
 525     args = parser.parse_args()
 526
 527     # Get normalized input parameters.
 528     source_dir = args.source_dir.absolute()
 529     background_image_filepath = get_background_image(args.background_image)
 530     applescript = get_applescript(args.applescript)
 531     codesign = args.codesign
 532
 533     app_bundles = collect_and_log_app_bundles(source_dir)
 534     if not app_bundles:
 535         return
 536
 537     dmg_filepath = get_dmg_filepath(args.dmg, app_bundles)
 538     volume_name = get_volume_name(args.volume_name, dmg_filepath)
 539
 540     print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)')
 541
 542     create_final_dmg(app_bundles,
 543                      dmg_filepath,
 544                      background_image_filepath,
 545                      volume_name,
 546                      applescript,
 547                      codesign)
 548
 549
 550 if __name__ == "__main__":
 551     main()