6929be8500300e51b07963ddb03f47130b2a12e0
[blender.git] / build_files / buildbot / slave_pack.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 # <pep8 compliant>
20
21 # Runs on buildbot slave, creating a release package using the build
22 # system and zipping it into buildbot_upload.zip. This is then uploaded
23 # to the master in the next buildbot step.
24
25 import os
26 import subprocess
27 import sys
28 import zipfile
29
30 # get builder name
31 if len(sys.argv) < 2:
32     sys.stderr.write("Not enough arguments, expecting builder name\n")
33     sys.exit(1)
34
35 builder = sys.argv[1]
36 # Never write branch if it is master.
37 branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
38
39 blender_dir = os.path.join('..', 'blender.git')
40 build_dir = os.path.join('..', 'build', builder)
41 install_dir = os.path.join('..', 'install', builder)
42 buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip"))
43
44 upload_filename = None  # Name of the archive to be uploaded
45                         # (this is the name of archive which will appear on the
46                         # download page)
47 upload_filepath = None  # Filepath to be uploaded to the server
48                         # (this folder will be packed)
49
50
51 def parse_header_file(filename, define):
52     import re
53     regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
54     with open(filename, "r") as file:
55         for l in file:
56             match = regex.match(l)
57             if match:
58                 return match.group(1)
59     return None
60
61
62 # Make sure install directory always exists
63 if not os.path.exists(install_dir):
64     os.makedirs(install_dir)
65
66
67 def create_tar_bz2(src, dest, package_name):
68     # One extra to remove leading os.sep when cleaning root for package_root
69     ln = len(src) + 1
70     flist = list()
71
72     # Create list of tuples containing file and archive name
73     for root, dirs, files in os.walk(src):
74         package_root = os.path.join(package_name, root[ln:])
75         flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
76
77     import tarfile
78     package = tarfile.open(dest, 'w:bz2')
79     for entry in flist:
80         package.add(entry[0], entry[1], recursive=False)
81     package.close()
82
83
84 if builder.find('cmake') != -1:
85     # CMake
86     if 'win' in builder or 'mac' in builder:
87         os.chdir(build_dir)
88
89         files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
90         for f in files:
91             os.remove(f)
92         retcode = subprocess.call(['cpack', '-G', 'ZIP'])
93         result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
94
95         # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
96         # in the CMake yet. For until then we do some magic here.
97         tokens = result_file.split('-')
98         blender_version = tokens[1].split('.')
99         blender_full_version = '.'.join(blender_version[0:2])
100         git_hash = tokens[2].split('.')[1]
101         platform = builder.split('_')[0]
102         if platform == 'mac':
103             # Special exception for OSX
104             platform = 'OSX-10.6-'
105             if builder.endswith('x86_64_10_6_cmake'):
106                 platform += 'x86_64'
107             elif builder.endswith('i386_10_6_cmake'):
108                 platform += 'i386'
109             elif builder.endswith('ppc_10_6_cmake'):
110                 platform += 'ppc'
111         if builder.endswith('vc2015'):
112             platform += "-vc14"
113         builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
114         # NOTE: Blender 2.8 is already respected by blender_full_version.
115         if branch != '' and branch != 'blender2.8':
116             builderified_name = branch + "-" + builderified_name
117
118         os.rename(result_file, "{}.zip".format(builderified_name))
119         # create zip file
120         try:
121             if os.path.exists(buildbot_upload_zip):
122                 os.remove(buildbot_upload_zip)
123             z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
124             z.write("{}.zip".format(builderified_name))
125             z.close()
126             sys.exit(retcode)
127         except Exception as ex:
128             sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
129             sys.exit(1)
130
131     elif builder.startswith('linux_'):
132         blender = os.path.join(install_dir, 'blender')
133         blenderplayer = os.path.join(install_dir, 'blenderplayer')
134
135         buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
136         blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
137
138         # Get version information
139         blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
140         blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
141         blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
142         blender_glibc = builder.split('_')[1]
143
144         if builder.endswith('x86_64_cmake'):
145             chroot_name = 'buildbot_squeeze_x86_64'
146             bits = 64
147             blender_arch = 'x86_64'
148         elif builder.endswith('i686_cmake'):
149             chroot_name = 'buildbot_squeeze_i686'
150             bits = 32
151             blender_arch = 'i686'
152
153         # Strip all unused symbols from the binaries
154         print("Stripping binaries...")
155         chroot_prefix = ['schroot', '-c', chroot_name, '--']
156         subprocess.call(chroot_prefix + ['strip', '--strip-all', blender, blenderplayer])
157
158         print("Stripping python...")
159         py_target = os.path.join(install_dir, blender_version)
160         subprocess.call(chroot_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
161
162         # Copy all specific files which are too specific to be copied by
163         # the CMake rules themselves
164         print("Copying extra scripts and libs...")
165
166         extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
167         mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
168         software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
169         icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
170
171         os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
172         os.system('cp %s %s' % (software_gl, install_dir))
173         os.system('cp -r %s %s' % (icons, install_dir))
174         os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
175
176         # Construct archive name
177         package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version,
178                                                       blender_hash,
179                                                       blender_glibc,
180                                                       blender_arch)
181         # NOTE: Blender 2.8 is already respected by blender_full_version.
182         if branch != '' and branch != 'blender2.8':
183             package_name = branch + "-" + package_name
184
185         upload_filename = package_name + ".tar.bz2"
186
187         print("Creating .tar.bz2 archive")
188         upload_filepath = install_dir + '.tar.bz2'
189         create_tar_bz2(install_dir, upload_filepath, package_name)
190 else:
191     print("Unknown building system")
192     sys.exit(1)
193
194
195 if upload_filepath is None:
196     # clean release directory if it already exists
197     release_dir = 'release'
198
199     if os.path.exists(release_dir):
200         for f in os.listdir(release_dir):
201             if os.path.isfile(os.path.join(release_dir, f)):
202                 os.remove(os.path.join(release_dir, f))
203
204     # create release package
205     try:
206         subprocess.call(['make', 'package_archive'])
207     except Exception as ex:
208         sys.stderr.write('Make package release failed' + str(ex) + '\n')
209         sys.exit(1)
210
211     # find release directory, must exist this time
212     if not os.path.exists(release_dir):
213         sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
214         sys.exit(1)
215
216     # find release package
217     file = None
218     filepath = None
219
220     for f in os.listdir(release_dir):
221         rf = os.path.join(release_dir, f)
222         if os.path.isfile(rf) and f.startswith('blender'):
223             file = f
224             filepath = rf
225
226     if not file:
227         sys.stderr.write("Failed to find release package.\n")
228         sys.exit(1)
229
230     upload_filename = file
231     upload_filepath = filepath
232
233 # create zip file
234 try:
235     upload_zip = os.path.join(buildbot_upload_zip)
236     if os.path.exists(upload_zip):
237         os.remove(upload_zip)
238     z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
239     z.write(upload_filepath, arcname=upload_filename)
240     z.close()
241 except Exception as ex:
242     sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
243     sys.exit(1)