Buildbot: Initial work to move linux build environment to CMake
[blender.git] / build_files / buildbot / slave_compile.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 # <pep8 compliant>
20
21 import os
22 import subprocess
23 import sys
24 import shutil
25
26 # get builder name
27 if len(sys.argv) < 2:
28     sys.stderr.write("Not enough arguments, expecting builder name\n")
29     sys.exit(1)
30
31 builder = sys.argv[1]
32
33 # we run from build/ directory
34 blender_dir = os.path.join('..', 'blender.git')
35
36 if 'cmake' in builder:
37     # cmake
38
39     # Some fine-tuning configuration
40     blender_dir = os.path.join('..', blender_dir)
41     build_dir = os.path.abspath(os.path.join('..', 'build', builder))
42     install_dir = os.path.abspath(os.path.join('..', 'install', builder))
43     targets = ['blender']
44
45     chroot_name = None  # If not None command will be delegated to that chroot
46     build_cubins = True  # Whether to build Cycles CUDA kernels
47     remove_cache = False  # Remove CMake cache to be sure config is totally up-to-date
48     remove_install_dir = False  # Remove installation folder before building
49
50     # Config file to be used (relative to blender's sources root)
51     cmake_config_file = "build_files/cmake/config/blender_full.cmake"
52     cmake_player_config_file = None
53     cmake_cuda_config_file = None
54
55     # Set build options.
56     cmake_options = ['-DCMAKE_BUILD_TYPE:STRING=Release']
57
58     if builder.startswith('mac'):
59         # Set up OSX architecture
60         if builder.endswith('x86_64_cmake'):
61             cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
62         elif builder.endswith('i386_cmake'):
63             cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386')
64         elif builder.endswith('ppc_cmake'):
65             cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc')
66
67     elif builder.startswith('win'):
68         if builder.startwith('win64'):
69             cmake_options.append(['-G', '"Visual Studio 12 2013 Win64"'])
70         elif builder.startswith('win32'):
71             cmake_options.append(['-G', '"Visual Studio 12 2013"'])
72             build_cubins = False
73
74     elif builder.startswith('linux'):
75         remove_cache = True
76         remove_install_dir = True
77         cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
78         cmake_player_config_file = "build_files/buildbot/config/blender_player_linux.cmake"
79         cmake_cuda_config_file = "build_files/buildbot/config/blender_cuda_linux.cmake"
80         if builder.endswith('x86_64_cmake'):
81             chroot_name = 'buildbot_squeeze_x86_64'
82             build_cubins = False
83             targets = ['player', 'blender']
84         elif builder.endswith('i386_cmake'):
85             chroot_name = 'buildbot_squeeze_i686'
86             build_cubins = False
87             targets = ['player', 'blender']
88
89     cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=%d" % (build_cubins))
90
91     if install_dir:
92         cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
93
94     cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
95
96     # Prepare chroot command prefix if needed
97
98     if chroot_name:
99         chroot_prefix = ['schroot', '-c', chroot_name, '--']
100     else:
101         chroot_prefix = []
102
103     # Make sure no garbage remained from the previous run
104     # (only do it if builder requested this)
105     if remove_install_dir:
106         if os.path.isdir(install_dir):
107             shutil.rmtree(install_dir)
108
109     for target in targets:
110         print("Building target %s" % (target))
111         # Construct build directory name based on the target
112         target_build_dir = build_dir
113         if target != 'blender':
114             target_build_dir += '_' + target
115         # Make sure build directory exists and enter it
116         if not os.path.isdir(target_build_dir):
117             os.mkdir(target_build_dir)
118         os.chdir(target_build_dir)
119         # Tweaking CMake options to respect the target
120         target_cmake_options = cmake_options[:]
121         if target == 'player':
122             target_cmake_options.append("-C" + os.path.join(blender_dir, cmake_player_config_file))
123         elif target == 'cuda':
124             target_cmake_options.append("-C" + os.path.join(blender_dir, cmake_cuda_config_file))
125         # Configure the build
126         print("CMake options:")
127         print(target_cmake_options)
128         if remove_cache and os.path.exists('CMakeCache.txt'):
129             print("Removing CMake cache")
130             os.remove('CMakeCache.txt')
131         retcode = subprocess.call(chroot_prefix + ['cmake', blender_dir] + target_cmake_options)
132         if retcode != 0:
133             print('Condifuration FAILED!')
134             sys.exit(retcode)
135
136         if 'win32' in builder:
137             command = ['msbuild', 'INSTALL.vcxproj', '/Property:PlatformToolset=v120_xp', '/p:Configuration=Release']
138         elif 'win64' in builder:
139             command = ['msbuild', 'INSTALL.vcxproj', '/p:Configuration=Release']
140         else:
141             command = chroot_prefix + ['make', '-s', '-j2', 'install']
142
143         print("Executing command:")
144         print(command)
145         retcode = subprocess.call(command)
146
147         if retcode != 0:
148             sys.exit(retcode)
149 else:
150     python_bin = 'python'
151     if builder.find('linux') != -1:
152         python_bin = '/opt/lib/python-2.7/bin/python2.7'
153
154     # scons
155     os.chdir(blender_dir)
156     scons_cmd = [python_bin, 'scons/scons.py']
157     scons_options = ['BF_FANCY=False']
158
159     # We're using the same rules as release builder, so tweak
160     # build and install dirs
161     build_dir = os.path.join('..', 'build', builder)
162     install_dir = os.path.join('..', 'install', builder)
163
164     # Clean install directory so we'll be sure there's no
165     # residual libs and files remained from the previous install.
166     if os.path.isdir(install_dir):
167         shutil.rmtree(install_dir)
168
169     buildbot_dir = os.path.dirname(os.path.realpath(__file__))
170     config_dir = os.path.join(buildbot_dir, 'config')
171
172     if builder.find('linux') != -1:
173         configs = []
174         if builder.endswith('linux_glibc211_x86_64_scons'):
175             configs = ['user-config-player-glibc211-x86_64.py',
176                        'user-config-cuda-glibc211-x86_64.py',
177                        'user-config-glibc211-x86_64.py'
178                        ]
179             chroot_name = 'buildbot_squeeze_x86_64'
180             cuda_chroot = 'buildbot_squeeze_x86_64'
181         elif builder.endswith('linux_glibc211_i386_scons'):
182             configs = ['user-config-player-glibc211-i686.py',
183                        'user-config-cuda-glibc211-i686.py',
184                        'user-config-glibc211-i686.py']
185             chroot_name = 'buildbot_squeeze_i686'
186
187             # use 64bit cuda toolkit, so there'll be no memory limit issues
188             cuda_chroot = 'buildbot_squeeze_x86_64'
189
190         # Compilation will happen inside of chroot environment
191         prog_scons_cmd = ['schroot', '-c', chroot_name, '--'] + scons_cmd
192         cuda_scons_cmd = ['schroot', '-c', cuda_chroot, '--'] + scons_cmd
193
194         common_options = ['BF_INSTALLDIR=' + install_dir] + scons_options
195
196         for config in configs:
197             config_fpath = os.path.join(config_dir, config)
198
199             scons_options = []
200
201             if config.find('player') != -1:
202                 scons_options.append('BF_BUILDDIR=%s_player' % (build_dir))
203             elif config.find('cuda') != -1:
204                 scons_options.append('BF_BUILDDIR=%s_cuda' % (build_dir))
205             else:
206                 scons_options.append('BF_BUILDDIR=%s' % (build_dir))
207
208             scons_options += common_options
209
210             if config.find('player') != -1:
211                 scons_options.append('blenderplayer')
212                 cur_scons_cmd = prog_scons_cmd
213             elif config.find('cuda') != -1:
214                 scons_options.append('cudakernels')
215                 cur_scons_cmd = cuda_scons_cmd
216
217                 if config.find('i686') != -1:
218                     scons_options.append('BF_BITNESS=32')
219                 elif config.find('x86_64') != -1:
220                     scons_options.append('BF_BITNESS=64')
221             else:
222                 scons_options.append('blender')
223                 cur_scons_cmd = prog_scons_cmd
224
225             scons_options.append('BF_CONFIG=' + config_fpath)
226
227             retcode = subprocess.call(cur_scons_cmd + scons_options)
228             if retcode != 0:
229                 print('Error building rules with config ' + config)
230                 sys.exit(retcode)
231
232         sys.exit(0)
233     else:
234         if builder.find('win') != -1:
235             bitness = '32'
236
237             if builder.find('win64') != -1:
238                 bitness = '64'
239
240             scons_options.append('BF_INSTALLDIR=' + install_dir)
241             scons_options.append('BF_BUILDDIR=' + build_dir)
242             scons_options.append('BF_BITNESS=' + bitness)
243             scons_options.append('WITH_BF_CYCLES_CUDA_BINARIES=True')
244             scons_options.append('BF_CYCLES_CUDA_NVCC=nvcc.exe')
245             if builder.find('mingw') != -1:
246                 scons_options.append('BF_TOOLSET=mingw')
247             if builder.endswith('vc2013'):
248                 scons_options.append('MSVS_VERSION=12.0')
249                 scons_options.append('MSVC_VERSION=12.0')
250                 scons_options.append('WITH_BF_CYCLES_CUDA_BINARIES=1')
251                 scons_options.append('BF_CYCLES_CUDA_NVCC=nvcc.exe')
252             scons_options.append('BF_NUMJOBS=1')
253
254         elif builder.find('mac') != -1:
255             if builder.find('x86_64') != -1:
256                 config = 'user-config-mac-x86_64.py'
257             else:
258                 config = 'user-config-mac-i386.py'
259
260             scons_options.append('BF_CONFIG=' + os.path.join(config_dir, config))
261
262         if builder.find('win') != -1:
263             if not os.path.exists(install_dir):
264                 os.makedirs(install_dir)
265             if builder.endswith('vc2013'):
266                 dlls = ('msvcp120.dll', 'msvcr120.dll', 'vcomp120.dll')
267             if builder.find('win64') == -1:
268                 dlls_path = '..\\..\\..\\redist\\x86'
269             else:
270                 dlls_path = '..\\..\\..\\redist\\amd64'
271             for dll in dlls:
272                 shutil.copyfile(os.path.join(dlls_path, dll), os.path.join(install_dir, dll))
273
274         retcode = subprocess.call([python_bin, 'scons/scons.py'] + scons_options)
275
276         sys.exit(retcode)