Merge branch 'blender-v2.92-release'
[blender.git] / build_files / buildbot / worker_bundle_dmg.py
1 #!/usr/bin/env python3
2
3 # ##### BEGIN GPL LICENSE BLOCK #####
4 #
5 #  This program is free software; you can redistribute it and/or
6 #  modify it under the terms of the GNU General Public License
7 #  as published by the Free Software Foundation; either version 2
8 #  of the License, or (at your option) any later version.
9 #
10 #  This program is distributed in the hope that it will be useful,
11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 #  GNU General Public License for more details.
14 #
15 #  You should have received a copy of the GNU General Public License
16 #  along with this program; if not, write to the Free Software Foundation,
17 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 #
19 # ##### END GPL LICENSE BLOCK #####
20
21 import argparse
22 import re
23 import shutil
24 import subprocess
25 import sys
26 import time
27
28 from pathlib import Path
29 from tempfile import TemporaryDirectory, NamedTemporaryFile
30 from typing import List
31
32 BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
33 CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py'
34 BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
35 DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
36
37
38 # Extra size which is added on top of actual files size when estimating size
39 # of destination DNG.
40 EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
41
42 ################################################################################
43 # Common utilities
44
45
46 def get_directory_size(root_directory: Path) -> int:
47     """
48     Get size of directory on disk
49     """
50
51     total_size = 0
52     for file in root_directory.glob('**/*'):
53         total_size += file.lstat().st_size
54     return total_size
55
56
57 ################################################################################
58 # DMG bundling specific logic
59
60 def create_argument_parser():
61     parser = argparse.ArgumentParser()
62     parser.add_argument(
63         'source_dir',
64         type=Path,
65         help='Source directory which points to either existing .app bundle'
66              'or to a directory with .app bundles.')
67     parser.add_argument(
68         '--background-image',
69         type=Path,
70         help="Optional background picture which will be set on the DMG."
71              "If not provided default Blender's one is used.")
72     parser.add_argument(
73         '--volume-name',
74         type=str,
75         help='Optional name of a volume which will be used for DMG.')
76     parser.add_argument(
77         '--dmg',
78         type=Path,
79         help='Optional argument which points to a final DMG file name.')
80     parser.add_argument(
81         '--applescript',
82         type=Path,
83         help="Optional path to applescript to set up folder looks of DMG."
84              "If not provided default Blender's one is used.")
85     parser.add_argument(
86         '--codesign',
87         action="store_true",
88         help="Code sign and notarize DMG contents.")
89     return parser
90
91
92 def collect_app_bundles(source_dir: Path) -> List[Path]:
93     """
94     Collect all app bundles which are to be put into DMG
95
96     If the source directory points to FOO.app it will be the only app bundle
97     packed.
98
99     Otherwise all .app bundles from given directory are placed to a single
100     DMG.
101     """
102
103     if source_dir.name.endswith('.app'):
104         return [source_dir]
105
106     app_bundles = []
107     for filename in source_dir.glob('*'):
108         if not filename.is_dir():
109             continue
110         if not filename.name.endswith('.app'):
111             continue
112
113         app_bundles.append(filename)
114
115     return app_bundles
116
117
118 def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
119     app_bundles = collect_app_bundles(source_dir)
120
121     if not app_bundles:
122         print('No app bundles found for packing')
123         return
124
125     print(f'Found {len(app_bundles)} to pack:')
126     for app_bundle in app_bundles:
127         print(f'- {app_bundle}')
128
129     return app_bundles
130
131
132 def estimate_dmg_size(app_bundles: List[Path]) -> int:
133     """
134     Estimate size of DMG to hold requested app bundles
135
136     The size is based on actual size of all files in all bundles plus some
137     space to compensate for different size-on-disk plus some space to hold
138     codesign signatures.
139
140     Is better to be on a high side since the empty space is compressed, but
141     lack of space might cause silent failures later on.
142     """
143
144     app_bundles_size = 0
145     for app_bundle in app_bundles:
146         app_bundles_size += get_directory_size(app_bundle)
147
148     return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
149
150
151 def copy_app_bundles_to_directory(app_bundles: List[Path],
152                                   directory: Path) -> None:
153     """
154     Copy all bundles to a given directory
155
156     This directory is what the DMG will be created from.
157     """
158     for app_bundle in app_bundles:
159         print(f'Copying {app_bundle.name}...')
160         shutil.copytree(app_bundle, directory / app_bundle.name)
161
162
163 def get_main_app_bundle(app_bundles: List[Path]) -> Path:
164     """
165     Get application bundle main for the installation
166     """
167     return app_bundles[0]
168
169
170 def create_dmg_image(app_bundles: List[Path],
171                      dmg_filepath: Path,
172                      volume_name: str) -> None:
173     """
174     Create DMG disk image and put app bundles in it
175
176     No DMG configuration or codesigning is happening here.
177     """
178
179     if dmg_filepath.exists():
180         print(f'Removing existing writable DMG {dmg_filepath}...')
181         dmg_filepath.unlink()
182
183     print('Preparing directory with app bundles for the DMG...')
184     with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
185         # Copy all bundles to a clean directory.
186         content_dir = Path(content_dir_str)
187         copy_app_bundles_to_directory(app_bundles, content_dir)
188
189         # Estimate size of the DMG.
190         dmg_size = estimate_dmg_size(app_bundles)
191         print(f'Estimated DMG size: {dmg_size:,} bytes.')
192
193         # Create the DMG.
194         print(f'Creating writable DMG {dmg_filepath}')
195         command = ('hdiutil',
196                    'create',
197                    '-size', str(dmg_size),
198                    '-fs', 'HFS+',
199                    '-srcfolder', content_dir,
200                    '-volname', volume_name,
201                    '-format', 'UDRW',
202                    dmg_filepath)
203         subprocess.run(command)
204
205
206 def get_writable_dmg_filepath(dmg_filepath: Path):
207     """
208     Get file path for writable DMG image
209     """
210     parent = dmg_filepath.parent
211     return parent / (dmg_filepath.stem + '-temp.dmg')
212
213
214 def mount_readwrite_dmg(dmg_filepath: Path) -> None:
215     """
216     Mount writable DMG
217
218     Mounting point would be /Volumes/<volume name>
219     """
220
221     print(f'Mounting read-write DMG ${dmg_filepath}')
222     command = ('hdiutil',
223                'attach', '-readwrite',
224                '-noverify',
225                '-noautoopen',
226                dmg_filepath)
227     subprocess.run(command)
228
229
230 def get_mount_directory_for_volume_name(volume_name: str) -> Path:
231     """
232     Get directory under which the volume will be mounted
233     """
234
235     return Path('/Volumes') / volume_name
236
237
238 def eject_volume(volume_name: str) -> None:
239     """
240     Eject given volume, if mounted
241     """
242     mount_directory = get_mount_directory_for_volume_name(volume_name)
243     if not mount_directory.exists():
244         return
245     mount_directory_str = str(mount_directory)
246
247     print(f'Ejecting volume {volume_name}')
248
249     # Figure out which device to eject.
250     mount_output = subprocess.check_output(['mount']).decode()
251     device = ''
252     for line in mount_output.splitlines():
253         if f'on {mount_directory_str} (' not in line:
254             continue
255         tokens = line.split(' ', 3)
256         if len(tokens) < 3:
257             continue
258         if tokens[1] != 'on':
259             continue
260         if device:
261             raise Exception(
262                 f'Multiple devices found for mounting point {mount_directory}')
263         device = tokens[0]
264
265     if not device:
266         raise Exception(
267             f'No device found for mounting point {mount_directory}')
268
269     print(f'{mount_directory} is mounted as device {device}, ejecting...')
270     subprocess.run(['diskutil', 'eject', device])
271
272
273 def copy_background_if_needed(background_image_filepath: Path,
274                               mount_directory: Path) -> None:
275     """
276     Copy background to the DMG
277
278     If the background image is not specified it will not be copied.
279     """
280
281     if not background_image_filepath:
282         print('No background image provided.')
283         return
284
285     print(f'Copying background image {background_image_filepath}')
286
287     destination_dir = mount_directory / '.background'
288     destination_dir.mkdir(exist_ok=True)
289
290     destination_filepath = destination_dir / background_image_filepath.name
291     shutil.copy(background_image_filepath, destination_filepath)
292
293
294 def create_applications_link(mount_directory: Path) -> None:
295     """
296     Create link to /Applications in the given location
297     """
298
299     print('Creating link to /Applications')
300
301     command = ('ln', '-s', '/Applications', mount_directory / ' ')
302     subprocess.run(command)
303
304
305 def run_applescript(applescript: Path,
306                     volume_name: str,
307                     app_bundles: List[Path],
308                     background_image_filepath: Path) -> None:
309     """
310     Run given applescript to adjust look and feel of the DMG
311     """
312
313     main_app_bundle = get_main_app_bundle(app_bundles)
314
315     with NamedTemporaryFile(
316             mode='w', suffix='.applescript') as temp_applescript:
317         print('Adjusting applescript for volume name...')
318         # Adjust script to the specific volume name.
319         with open(applescript, mode='r') as input:
320             for line in input.readlines():
321                 stripped_line = line.strip()
322                 if stripped_line.startswith('tell disk'):
323                     line = re.sub('tell disk ".*"',
324                                   f'tell disk "{volume_name}"',
325                                   line)
326                 elif stripped_line.startswith('set background picture'):
327                     if not background_image_filepath:
328                         continue
329                     else:
330                         background_image_short = \
331                             '.background:' + background_image_filepath.name
332                         line = re.sub('to file ".*"',
333                                       f'to file "{background_image_short}"',
334                                       line)
335                 line = line.replace('blender.app', main_app_bundle.name)
336                 temp_applescript.write(line)
337
338         temp_applescript.flush()
339
340         print('Running applescript...')
341         command = ('osascript',  temp_applescript.name)
342         subprocess.run(command)
343
344         print('Waiting for applescript...')
345
346         # NOTE: This is copied from bundle.sh. The exact reason for sleep is
347         # still remained a mystery.
348         time.sleep(5)
349
350
351 def codesign(subject: Path):
352     """
353     Codesign file or directory
354
355     NOTE: For DMG it will also notarize.
356     """
357
358     command = (CODESIGN_SCRIPT, subject)
359     subprocess.run(command)
360
361
362 def codesign_app_bundles_in_dmg(mount_directory: str) -> None:
363     """
364     Code sign all binaries and bundles in the mounted directory
365     """
366
367     print(f'Codesigning all app bundles in {mount_directory}')
368     codesign(mount_directory)
369
370
371 def codesign_and_notarize_dmg(dmg_filepath: Path) -> None:
372     """
373     Run codesign and notarization pipeline on the DMG
374     """
375
376     print(f'Codesigning and notarizing DMG {dmg_filepath}')
377     codesign(dmg_filepath)
378
379
380 def compress_dmg(writable_dmg_filepath: Path,
381                  final_dmg_filepath: Path) -> None:
382     """
383     Compress temporary read-write DMG
384     """
385     command = ('hdiutil', 'convert',
386                writable_dmg_filepath,
387                '-format', 'UDZO',
388                '-o', final_dmg_filepath)
389
390     if final_dmg_filepath.exists():
391         print(f'Removing old compressed DMG {final_dmg_filepath}')
392         final_dmg_filepath.unlink()
393
394     print('Compressing disk image...')
395     subprocess.run(command)
396
397
398 def create_final_dmg(app_bundles: List[Path],
399                      dmg_filepath: Path,
400                      background_image_filepath: Path,
401                      volume_name: str,
402                      applescript: Path,
403                      codesign: bool) -> None:
404     """
405     Create DMG with all app bundles
406
407     Will take care configuring background, signing all binaries and app bundles
408     and notarizing the DMG.
409     """
410
411     print('Running all routines to create final DMG')
412
413     writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath)
414     mount_directory = get_mount_directory_for_volume_name(volume_name)
415
416     # Make sure volume is not mounted.
417     # If it is mounted it will prevent removing old DMG files and could make
418     # it so app bundles are copied to the wrong place.
419     eject_volume(volume_name)
420
421     create_dmg_image(app_bundles, writable_dmg_filepath, volume_name)
422
423     mount_readwrite_dmg(writable_dmg_filepath)
424
425     # Run codesign first, prior to copying amything else.
426     #
427     # This allows to recurs into the content of bundles without worrying about
428     # possible interfereice of Application symlink.
429     if codesign:
430         codesign_app_bundles_in_dmg(mount_directory)
431
432     copy_background_if_needed(background_image_filepath, mount_directory)
433     create_applications_link(mount_directory)
434     run_applescript(applescript, volume_name, app_bundles,
435                     background_image_filepath)
436
437     print('Ejecting read-write DMG image...')
438     eject_volume(volume_name)
439
440     compress_dmg(writable_dmg_filepath, dmg_filepath)
441     writable_dmg_filepath.unlink()
442
443     if codesign:
444         codesign_and_notarize_dmg(dmg_filepath)
445
446
447 def ensure_dmg_extension(filepath: Path) -> Path:
448     """
449     Make sure given file have .dmg extension
450     """
451
452     if filepath.suffix != '.dmg':
453         return filepath.with_suffix(f'{filepath.suffix}.dmg')
454     return filepath
455
456
457 def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path:
458     """
459     Get full file path for the final DMG image
460
461     Will use the provided one when possible, otherwise will deduct it from
462     app bundles.
463
464     If the name is deducted, the DMG is stored in the current directory.
465     """
466
467     if requested_name:
468         return ensure_dmg_extension(requested_name.absolute())
469
470     # TODO(sergey): This is not necessarily the main one.
471     main_bundle = app_bundles[0]
472     # Strip .app from the name
473     return Path(main_bundle.name[:-4] + '.dmg').absolute()
474
475
476 def get_background_image(requested_background_image: Path) -> Path:
477     """
478     Get effective filepath for the background image
479     """
480
481     if requested_background_image:
482         return requested_background_image.absolute()
483
484     return DARWIN_DIRECTORY / 'background.tif'
485
486
487 def get_applescript(requested_applescript: Path) -> Path:
488     """
489     Get effective filepath for the applescript
490     """
491
492     if requested_applescript:
493         return requested_applescript.absolute()
494
495     return DARWIN_DIRECTORY / 'blender.applescript'
496
497
498 def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str:
499     """
500     Deduct volume name from the DMG path
501
502     Will use first part of the DMG file name prior to dash.
503     """
504
505     tokens = dmg_filepath.stem.split('-')
506     words = tokens[0].split()
507
508     return ' '.join(word.capitalize() for word in words)
509
510
511 def get_volume_name(requested_volume_name: str,
512                     dmg_filepath: Path) -> str:
513     """
514     Get effective name for DMG volume
515     """
516
517     if requested_volume_name:
518         return requested_volume_name
519
520     return get_volume_name_from_dmg_filepath(dmg_filepath)
521
522
523 def main():
524     parser = create_argument_parser()
525     args = parser.parse_args()
526
527     # Get normalized input parameters.
528     source_dir = args.source_dir.absolute()
529     background_image_filepath = get_background_image(args.background_image)
530     applescript = get_applescript(args.applescript)
531     codesign = args.codesign
532
533     app_bundles = collect_and_log_app_bundles(source_dir)
534     if not app_bundles:
535         return
536
537     dmg_filepath = get_dmg_filepath(args.dmg, app_bundles)
538     volume_name = get_volume_name(args.volume_name, dmg_filepath)
539
540     print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)')
541
542     create_final_dmg(app_bundles,
543                      dmg_filepath,
544                      background_image_filepath,
545                      volume_name,
546                      applescript,
547                      codesign)
548
549
550 if __name__ == "__main__":
551     main()