Merge branch 'blender2.7'
[blender.git] / build_files / buildbot / slave_compile.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 # <pep8 compliant>
20
21 import os
22 import subprocess
23 import sys
24 import shutil
25
26 # get builder name
27 if len(sys.argv) < 2:
28     sys.stderr.write("Not enough arguments, expecting builder name\n")
29     sys.exit(1)
30
31 builder = sys.argv[1]
32
33 # we run from build/ directory
34 blender_dir = os.path.join('..', 'blender.git')
35
36
37 def parse_header_file(filename, define):
38     import re
39     regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
40     with open(filename, "r") as file:
41         for l in file:
42             match = regex.match(l)
43             if match:
44                 return match.group(1)
45     return None
46
47 if 'cmake' in builder:
48     # cmake
49
50     # Some fine-tuning configuration
51     blender_dir = os.path.abspath(blender_dir)
52     build_dir = os.path.abspath(os.path.join('..', 'build', builder))
53     install_dir = os.path.abspath(os.path.join('..', 'install', builder))
54     targets = ['blender']
55
56     chroot_name = None  # If not None command will be delegated to that chroot
57     cuda_chroot_name = None  # If not None cuda compilationcommand will be delegated to that chroot
58     build_cubins = True  # Whether to build Cycles CUDA kernels
59     bits = 64
60
61     # Config file to be used (relative to blender's sources root)
62     cmake_config_file = "build_files/cmake/config/blender_release.cmake"
63     cmake_cuda_config_file = None
64
65     # Set build options.
66     cmake_options = []
67     cmake_extra_options = ['-DCMAKE_BUILD_TYPE:STRING=Release']
68     cuda_cmake_options = []
69
70     if builder.startswith('mac'):
71         # Set up OSX architecture
72         if builder.endswith('x86_64_10_9_cmake'):
73             cmake_extra_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
74         cmake_extra_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
75
76     elif builder.startswith('win'):
77         if builder.endswith('_vs2017'):
78             if builder.startswith('win64'):
79                 cmake_options.extend(['-G', 'Visual Studio 15 2017 Win64'])
80             elif builder.startswith('win32'):
81                 bits = 32
82                 cmake_options.extend(['-G', 'Visual Studio 15 2017'])
83         elif builder.endswith('_vc2015'):
84             if builder.startswith('win64'):
85                 cmake_options.extend(['-G', 'Visual Studio 14 2015 Win64'])
86             elif builder.startswith('win32'):
87                 bits = 32
88                 cmake_options.extend(['-G', 'Visual Studio 14 2015'])
89             cmake_extra_options.append('-DCUDA_NVCC_FLAGS=--cl-version;2013;' +
90                 '--compiler-bindir;C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\bin')
91         else:
92             if builder.startswith('win64'):
93                 cmake_options.extend(['-G', 'Visual Studio 12 2013 Win64'])
94             elif builder.startswith('win32'):
95                 bits = 32
96                 cmake_options.extend(['-G', 'Visual Studio 12 2013'])
97
98     elif builder.startswith('linux'):
99         tokens = builder.split("_")
100         glibc = tokens[1]
101         if glibc == 'glibc224':
102             deb_name = "stretch"
103         elif glibc == 'glibc219':
104             deb_name = "jessie"
105         elif glibc == 'glibc211':
106             deb_name = "squeeze"
107         cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
108         if builder.endswith('x86_64_cmake'):
109             chroot_name = 'buildbot_' + deb_name + '_x86_64'
110             targets = ['blender']
111         elif builder.endswith('i686_cmake'):
112             bits = 32
113             chroot_name = 'buildbot_' + deb_name + '_i686'
114             targets = ['blender']
115         if deb_name != "stretch":
116             cmake_extra_options.extend(["-DCMAKE_C_COMPILER=/usr/bin/gcc-7",
117                                         "-DCMAKE_CXX_COMPILER=/usr/bin/g++-7"])
118
119     # Workaround to build only sm_7x kernels with CUDA 10, until
120     # older kernels work well with this version.
121     if builder.startswith('win'):
122         cmake_extra_options.append('-DCUDA_VERSION=9.1')
123         cmake_extra_options.append('-DCUDA_TOOLKIT_INCLUDE:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/include')
124         cmake_extra_options.append('-DCUDA_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1')
125         cmake_extra_options.append('-DCUDA_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/bin/nvcc.exe')
126         cmake_extra_options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0/bin/nvcc.exe')
127         cmake_extra_options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0')
128     elif builder.startswith('linux'):
129         cmake_extra_options.append('-DCUDA_VERSION=9.1')
130         cmake_extra_options.append('-DCUDA_TOOLKIT_INCLDUE:PATH=/usr/local/cuda-9.1/include')
131         cmake_extra_options.append('-DCUDA_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-9.1')
132         cmake_extra_options.append('-DCUDA_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-9.1/bin/nvcc')
133         cmake_extra_options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.0/bin/nvcc')
134         cmake_extra_options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.0')
135
136     cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
137
138     # Prepare CMake options needed to configure cuda binaries compilation, 64bit only.
139     if bits == 64:
140         cuda_cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=%s" % ('ON' if build_cubins else 'OFF'))
141         if build_cubins or 'cuda' in targets:
142             cuda_cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON")
143
144         # Only modify common cmake options if cuda doesn't require separate target.
145         if 'cuda' not in targets:
146             cmake_options += cuda_cmake_options
147     else:
148         cuda_cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF")
149
150     cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
151
152     cmake_options += cmake_extra_options
153
154     # Prepare chroot command prefix if needed
155     if chroot_name:
156         chroot_prefix = ['schroot', '-c', chroot_name, '--']
157     else:
158         chroot_prefix = []
159     if cuda_chroot_name:
160         cuda_chroot_prefix = ['schroot', '-c', cuda_chroot_name, '--']
161     else:
162         cuda_chroot_prefix = chroot_prefix[:]
163
164     # Make sure no garbage remained from the previous run
165     if os.path.isdir(install_dir):
166         shutil.rmtree(install_dir)
167
168     for target in targets:
169         print("Building target %s" % (target))
170         # Construct build directory name based on the target
171         target_build_dir = build_dir
172         target_chroot_prefix = chroot_prefix[:]
173         if target != 'blender':
174             target_build_dir += '_' + target
175         target_name = 'install'
176         # Tweaking CMake options to respect the target
177         target_cmake_options = cmake_options[:]
178         if target == 'cuda':
179             target_cmake_options += cuda_cmake_options
180             target_chroot_prefix = cuda_chroot_prefix[:]
181             target_name = 'cycles_kernel_cuda'
182         # If cuda binaries are compiled as a separate target, make sure
183         # other targets don't compile cuda binaries.
184         if 'cuda' in targets and target != 'cuda':
185             target_cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF")
186         # Do extra git fetch because not all platform/git/buildbot combinations
187         # update the origin remote, causing buildinfo to detect local changes.
188         os.chdir(blender_dir)
189         print("Fetching remotes")
190         command = ['git', 'fetch', '--all']
191         print(command)
192         retcode = subprocess.call(target_chroot_prefix + command)
193         if retcode != 0:
194             sys.exit(retcode)
195         # Make sure build directory exists and enter it
196         if not os.path.isdir(target_build_dir):
197             os.mkdir(target_build_dir)
198         os.chdir(target_build_dir)
199         # Configure the build
200         print("CMake options:")
201         print(target_cmake_options)
202         if os.path.exists('CMakeCache.txt'):
203             print("Removing CMake cache")
204             os.remove('CMakeCache.txt')
205         retcode = subprocess.call(target_chroot_prefix + ['cmake', blender_dir] + target_cmake_options)
206         if retcode != 0:
207             print('Configuration FAILED!')
208             sys.exit(retcode)
209
210         if 'win32' in builder or 'win64' in builder:
211             command = ['cmake', '--build', '.', '--target', target_name, '--config', 'Release']
212         else:
213             command = ['make', '-s', '-j2', target_name]
214
215         print("Executing command:")
216         print(command)
217         retcode = subprocess.call(target_chroot_prefix + command)
218
219         if retcode != 0:
220             sys.exit(retcode)
221
222         if builder.startswith('linux') and target == 'cuda':
223             blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
224             blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
225             blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
226             kernels = os.path.join(target_build_dir, 'intern', 'cycles', 'kernel')
227             install_kernels = os.path.join(install_dir, blender_version, 'scripts', 'addons', 'cycles', 'lib')
228             os.mkdir(install_kernels)
229             print("Copying cuda binaries from %s to %s" % (kernels, install_kernels))
230             os.system('cp %s/*.cubin %s' % (kernels, install_kernels))
231
232 else:
233     print("Unknown building system")
234     sys.exit(1)