Buildbot: Attempt to fix cpack target on windows
[blender.git] / build_files / buildbot / slave_pack.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 # <pep8 compliant>
20
21 # Runs on buildbot slave, creating a release package using the build
22 # system and zipping it into buildbot_upload.zip. This is then uploaded
23 # to the master in the next buildbot step.
24
25 import os
26 import subprocess
27 import sys
28 import zipfile
29
30 # get builder name
31 if len(sys.argv) < 2:
32     sys.stderr.write("Not enough arguments, expecting builder name\n")
33     sys.exit(1)
34
35 builder = sys.argv[1]
36 # Never write branch if it is master.
37 branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
38
39 upload_filename = None  # Name of the archive to be uploaded
40                         # (this is the name of archive which will appear on the
41                         # download page)
42 upload_filepath = None  # Filepath to be uploaded to the server
43                         # (this folder will be packed)
44
45
46 def parse_header_file(filename, define):
47     import re
48     regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
49     with open(filename, "r") as file:
50         for l in file:
51             match = regex.match(l)
52             if match:
53                 return match.group(1)
54     return None
55
56
57 # scons does own packaging
58 if builder.find('scons') != -1:
59     python_bin = 'python'
60     if builder.find('linux') != -1:
61         python_bin = '/opt/lib/python-2.7/bin/python2.7'
62
63     os.chdir('../blender.git')
64     scons_options = ['BF_QUICK=slnt', 'BUILDBOT_BRANCH=' + branch, 'buildslave', 'BF_FANCY=False']
65
66     buildbot_dir = os.path.dirname(os.path.realpath(__file__))
67     config_dir = os.path.join(buildbot_dir, 'config')
68     build_dir = os.path.join('..', 'build', builder)
69     install_dir = os.path.join('..', 'install', builder)
70
71     if builder.find('linux') != -1:
72         scons_options += ['WITH_BF_NOBLENDER=True', 'WITH_BF_PLAYER=False',
73                           'BF_BUILDDIR=' + build_dir,
74                           'BF_INSTALLDIR=' + install_dir,
75                           'WITHOUT_BF_INSTALL=True']
76
77         config = None
78         bits = None
79
80         if builder.endswith('linux_glibc211_x86_64_scons'):
81             config = 'user-config-glibc211-x86_64.py'
82             chroot_name = 'buildbot_squeeze_x86_64'
83             bits = 64
84         elif builder.endswith('linux_glibc211_i386_scons'):
85             config = 'user-config-glibc211-i686.py'
86             chroot_name = 'buildbot_squeeze_i686'
87             bits = 32
88
89         if config is not None:
90             config_fpath = os.path.join(config_dir, config)
91             scons_options.append('BF_CONFIG=' + config_fpath)
92
93         blender = os.path.join(install_dir, 'blender')
94         blenderplayer = os.path.join(install_dir, 'blenderplayer')
95         subprocess.call(['schroot', '-c', chroot_name, '--', 'strip', '--strip-all', blender, blenderplayer])
96
97         extra = "/home/sources/release-builder/extra/"
98         mesalibs = os.path.join(extra, 'mesalibs%d.tar.bz2' % bits)
99         software_gl = os.path.join(extra, 'blender-softwaregl')
100
101         os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
102         os.system('cp %s %s' % (software_gl, install_dir))
103         os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
104
105         retcode = subprocess.call(['schroot', '-c', chroot_name, '--', python_bin, 'scons/scons.py'] + scons_options)
106
107         sys.exit(retcode)
108     else:
109         if builder.find('win') != -1:
110             bitness = '32'
111
112             if builder.find('win64') != -1:
113                 bitness = '64'
114
115             scons_options.append('BF_INSTALLDIR=' + install_dir)
116             scons_options.append('BF_BUILDDIR=' + build_dir)
117             scons_options.append('BF_BITNESS=' + bitness)
118             scons_options.append('WITH_BF_CYCLES_CUDA_BINARIES=True')
119             scons_options.append('BF_CYCLES_CUDA_NVCC=nvcc.exe')
120             if builder.find('mingw') != -1:
121                 scons_options.append('BF_TOOLSET=mingw')
122             if builder.endswith('vc2013'):
123                 scons_options.append('MSVS_VERSION=12.0')
124                 scons_options.append('MSVC_VERSION=12.0')
125
126         elif builder.find('mac') != -1:
127             if builder.find('x86_64') != -1:
128                 config = 'user-config-mac-x86_64.py'
129             else:
130                 config = 'user-config-mac-i386.py'
131
132             scons_options.append('BF_CONFIG=' + os.path.join(config_dir, config))
133
134         retcode = subprocess.call([python_bin, 'scons/scons.py'] + scons_options)
135         sys.exit(retcode)
136 else:
137     # CMake
138     if 'win' in builder:
139         build_dir = os.path.join('..', 'build', builder)
140         os.chdir(build_dir)
141
142         files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
143         for f in files:
144             os.remove(f)
145         retcode = subprocess.call(['cpack', '-G', 'ZIP'])
146         result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
147
148         # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
149         # in the CMake yet. For until then we do some magic here.
150         tokens = result_file.split('-')
151         blender_version = tokens[1].split('.')
152         blender_full_version = '.'.join(blender_version[0:2])
153         git_hash = tokens[2].split('.')[1]
154         platform = builder.split('_')[0]
155         builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
156         if branch != '':
157             builderified_name = branch + "-" + builderified_name
158
159         os.rename(result_file, "{}.zip".format(builderified_name))
160         # create zip file
161         try:
162             upload_zip = "buildbot_upload.zip"
163             if os.path.exists(upload_zip):
164                 os.remove(upload_zip)
165             z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
166             z.write("{}.zip".format(builderified_name))
167             z.close()
168             sys.exit(retcode)
169         except Exception as ex:
170             sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
171             sys.exit(1)
172
173     elif builder.startswith('linux_'):
174         blender_dir = os.path.join('..', 'blender.git')
175         build_dir = os.path.join('..', 'build', builder)
176         install_dir = os.path.join('..', 'install', builder)
177
178         blender = os.path.join(install_dir, 'blender')
179         blenderplayer = os.path.join(install_dir, 'blenderplayer')
180
181         buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
182         blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender.h")
183
184         if builder.endswith('x86_64_cmake'):
185             chroot_name = 'buildbot_squeeze_x86_64'
186             bits = 64
187             blender_arch = 'x86_64'
188         elif builder.endswith('i686_cmake'):
189             chroot_name = 'buildbot_squeeze_i686'
190             bits = 32
191             blender_arch = 'i686'
192
193         # Strip all unused symbols from the binaries
194         print("Stripping binaries...")
195         chroot_prefix = ['schroot', '-c', chroot_name, '--']
196         subprocess.call(chroot_prefix + ['strip', '--strip-all', blender, blenderplayer])
197
198         # Copy all specific files which are too specific to be copied by
199         # the CMake rules themselves
200         print("Copying extra scripts and libs...")
201
202         extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
203         mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
204         software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
205         icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
206
207         os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
208         os.system('cp %s %s' % (software_gl, install_dir))
209         os.system('cp -r %s %s' % (icons, install_dir))
210         os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
211
212         # Get version information for the archive name
213         blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
214         blender_version = "%d.%d" % (blender_version / 100, blender_version % 100)
215         blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
216         blender_glibc = builder.split('_')[1]
217
218         upload_filename = 'blender-%s-%s-linux-%s-%s.tar.bz2' % (blender_version,
219                                                                  blender_hash,
220                                                                  blender_glibc,
221                                                                  blender_arch)
222         if branch != '':
223             upload_filename = branch + "-" + upload_filename
224
225         print("Creating .tar.bz2 archive")
226         os.system('tar -C../install -cjf %s.tar.bz2 %s' % (builder, builder))
227         upload_filepath = install_dir + '.tar.bz2'
228
229
230 if upload_filepath is None:
231     # clean release directory if it already exists
232     release_dir = 'release'
233
234     if os.path.exists(release_dir):
235         for f in os.listdir(release_dir):
236             if os.path.isfile(os.path.join(release_dir, f)):
237                 os.remove(os.path.join(release_dir, f))
238
239     # create release package
240     try:
241         subprocess.call(['make', 'package_archive'])
242     except Exception as ex:
243         sys.stderr.write('Make package release failed' + str(ex) + '\n')
244         sys.exit(1)
245
246     # find release directory, must exist this time
247     if not os.path.exists(release_dir):
248         sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
249         sys.exit(1)
250
251     # find release package
252     file = None
253     filepath = None
254
255     for f in os.listdir(release_dir):
256         rf = os.path.join(release_dir, f)
257         if os.path.isfile(rf) and f.startswith('blender'):
258             file = f
259             filepath = rf
260
261     if not file:
262         sys.stderr.write("Failed to find release package.\n")
263         sys.exit(1)
264
265     upload_filename = file
266     upload_filepath = filepath
267
268 # create zip file
269 try:
270     upload_zip = "buildbot_upload.zip"
271     if os.path.exists(upload_zip):
272         os.remove(upload_zip)
273     z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
274     z.write(upload_filepath, arcname=upload_filename)
275     z.close()
276 except Exception as ex:
277     sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
278     sys.exit(1)