731996df8ef7a20bc0b45b0365c325a4dc322246
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import glob
6 import os
7 import pathlib
8 import shlex
9 import shutil
10 import subprocess
11 import sys
12 import time
13 import tempfile
14
15
16 class COLORS_ANSI:
17     RED = '\033[00;31m'
18     GREEN = '\033[00;32m'
19     ENDC = '\033[0m'
20
21
22 class COLORS_DUMMY:
23     RED = ''
24     GREEN = ''
25     ENDC = ''
26
27 COLORS = COLORS_DUMMY
28
29
30 def print_message(message, type=None, status=''):
31     if type == 'SUCCESS':
32         print(COLORS.GREEN, end="")
33     elif type == 'FAILURE':
34         print(COLORS.RED, end="")
35     status_text = ...
36     if status == 'RUN':
37         status_text = " RUN      "
38     elif status == 'OK':
39         status_text = "       OK "
40     elif status == 'PASSED':
41         status_text = "  PASSED  "
42     elif status == 'FAILED':
43         status_text = "  FAILED  "
44     else:
45         status_text = status
46     if status_text:
47         print("[{}]" . format(status_text), end="")
48     print(COLORS.ENDC, end="")
49     print(" {}" . format(message))
50     sys.stdout.flush()
51
52
53 def render_file(filepath):
54     dirname = os.path.dirname(filepath)
55     basedir = os.path.dirname(dirname)
56     subject = os.path.basename(dirname)
57
58     custom_args = os.getenv('CYCLESTEST_ARGS')
59     custom_args = shlex.split(custom_args) if custom_args else []
60
61     # OSL and GPU examples
62     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
63     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
64
65     if subject == 'opengl':
66         command = [
67             BLENDER,
68             "--window-geometry", "0", "0", "1", "1",
69             "-noaudio",
70             "--factory-startup",
71             "--enable-autoexec",
72             filepath,
73             "-E", "CYCLES"]
74         command += custom_args
75         command += [
76             "-o", TEMP_FILE_MASK,
77             "-F", "PNG",
78             '--python', os.path.join(basedir,
79                                      "util",
80                                      "render_opengl.py")]
81     elif subject == 'bake':
82         command = [
83             BLENDER,
84             "-b",
85             "-noaudio",
86             "--factory-startup",
87             "--enable-autoexec",
88             filepath,
89             "-E", "CYCLES"]
90         command += custom_args
91         command += [
92             "-o", TEMP_FILE_MASK,
93             "-F", "PNG",
94             '--python', os.path.join(basedir,
95                                      "util",
96                                      "render_bake.py")]
97     else:
98         command = [
99             BLENDER,
100             "--background",
101             "-noaudio",
102             "--factory-startup",
103             "--enable-autoexec",
104             filepath,
105             "-E", "CYCLES"]
106         command += custom_args
107         command += [
108             "-o", TEMP_FILE_MASK,
109             "-F", "PNG",
110             "-f", "1"]
111     try:
112         output = subprocess.check_output(command)
113         if VERBOSE:
114             print(output.decode("utf-8"))
115         return None
116     except subprocess.CalledProcessError as e:
117         if os.path.exists(TEMP_FILE):
118             os.remove(TEMP_FILE)
119         if VERBOSE:
120             print(e.output.decode("utf-8"))
121         if b"Error: engine not found" in e.output:
122             return "NO_CYCLES"
123         elif b"blender probably wont start" in e.output:
124             return "NO_START"
125         return "CRASH"
126     except BaseException as e:
127         if os.path.exists(TEMP_FILE):
128             os.remove(TEMP_FILE)
129         if VERBOSE:
130             print(e)
131         return "CRASH"
132
133
134 def test_get_name(filepath):
135     filename = os.path.basename(filepath)
136     return os.path.splitext(filename)[0]
137
138 def test_get_images(filepath):
139     testname = test_get_name(filepath)
140     dirpath = os.path.dirname(filepath)
141
142     old_dirpath = os.path.join(dirpath, "reference_renders")
143     old_img = os.path.join(old_dirpath, testname + ".png")
144
145     ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref")
146     ref_img = os.path.join(ref_dirpath, testname + ".png")
147     if not os.path.exists(ref_dirpath):
148         os.makedirs(ref_dirpath)
149     if os.path.exists(old_img):
150         shutil.copy(old_img, ref_img)
151
152     new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
153     if not os.path.exists(new_dirpath):
154         os.makedirs(new_dirpath)
155     new_img = os.path.join(new_dirpath, testname + ".png")
156
157     diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
158     if not os.path.exists(diff_dirpath):
159         os.makedirs(diff_dirpath)
160     diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
161
162     return old_img, ref_img, new_img, diff_img
163
164
165 class Report:
166     def __init__(self, testname):
167         self.failed_tests = ""
168         self.passed_tests = ""
169         self.testname = testname
170
171     def output(self):
172         # write intermediate data for single test
173         outdir = os.path.join(OUTDIR, self.testname)
174         if not os.path.exists(outdir):
175             os.makedirs(outdir)
176
177         filepath = os.path.join(outdir, "failed.data")
178         pathlib.Path(filepath).write_text(self.failed_tests)
179
180         filepath = os.path.join(outdir, "passed.data")
181         pathlib.Path(filepath).write_text(self.passed_tests)
182
183         # gather intermediate data for all tests
184         failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
185         passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
186
187         failed_tests = ""
188         passed_tests = ""
189
190         for filename in failed_data:
191             filepath = os.path.join(OUTDIR, filename)
192             failed_tests += pathlib.Path(filepath).read_text()
193         for filename in passed_data:
194             filepath = os.path.join(OUTDIR, filename)
195             passed_tests += pathlib.Path(filepath).read_text()
196
197         # write html for all tests
198         self.html = """
199 <html>
200 <head>
201     <title>Cycles Test Report</title>
202     <style>
203         img {{ image-rendering: pixelated; width: 256px; background-color: #000; }}
204         img.render {{
205             background-color: #fff;
206             background-image:
207               -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
208               -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
209               -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
210               -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
211             background-image:
212               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
213               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
214               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
215               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
216
217             -moz-background-size:50px 50px;
218             background-size:50px 50px;
219             -webkit-background-size:50px 51px; /* override value for shitty webkit */
220
221             background-position:0 0, 25px 0, 25px -25px, 0px 25px;
222         }}
223         table td:first-child {{ width: 256px; }}
224     </style>
225     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
226 </head>
227 <body>
228     <div class="container">
229         <br/>
230         <h1>Cycles Test Report</h1>
231         <br/>
232         <table class="table table-striped">
233             <thead class="thead-default">
234                 <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
235             </thead>
236             {}{}
237         </table>
238         <br/>
239     </div>
240 </body>
241 </html>
242             """ . format(failed_tests, passed_tests)
243
244         filepath = os.path.join(OUTDIR, "report.html")
245         pathlib.Path(filepath).write_text(self.html)
246
247         print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
248
249     def relative_url(self, filepath):
250         relpath = os.path.relpath(filepath, OUTDIR)
251         return pathlib.Path(relpath).as_posix()
252
253     def add_test(self, filepath, error):
254         name = test_get_name(filepath)
255         name = name.replace('_', ' ')
256
257         old_img, ref_img, new_img, diff_img = test_get_images(filepath)
258
259         status = error if error else ""
260         style = """ style="background-color: #f99;" """ if error else ""
261
262         new_url = self.relative_url(new_img)
263         ref_url = self.relative_url(ref_img)
264         diff_url = self.relative_url(diff_img)
265
266         test_html = """
267             <tr{}>
268                 <td><b>{}</b><br/>{}<br/>{}</td>
269                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
270                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
271                 <td><img src="{}"></td>
272             </tr>""" . format(style, name, self.testname, status,
273                               new_url, ref_url, new_url,
274                               ref_url, new_url, ref_url,
275                               diff_url)
276
277         if error:
278             self.failed_tests += test_html
279         else:
280             self.passed_tests += test_html
281
282
283 def verify_output(report, filepath):
284     old_img, ref_img, new_img, diff_img = test_get_images(filepath)
285
286     # copy new image
287     if os.path.exists(new_img):
288         os.remove(new_img)
289     if os.path.exists(TEMP_FILE):
290         shutil.copy(TEMP_FILE, new_img)
291
292     update = os.getenv('CYCLESTEST_UPDATE')
293
294     if os.path.exists(ref_img):
295         # diff test with threshold
296         command = (
297             IDIFF,
298             "-fail", "0.016",
299             "-failpercent", "1",
300             ref_img,
301             TEMP_FILE,
302             )
303         try:
304             subprocess.check_output(command)
305             failed = False
306         except subprocess.CalledProcessError as e:
307             if VERBOSE:
308                 print_message(e.output.decode("utf-8"))
309             failed = e.returncode != 1
310     else:
311         if not update:
312             return False
313
314         failed = True
315
316     if failed and update:
317         # update reference
318         shutil.copy(new_img, ref_img)
319         shutil.copy(new_img, old_img)
320         failed = False
321
322     # generate diff image
323     command = (
324         IDIFF,
325         "-o", diff_img,
326         "-abs", "-scale", "16",
327         ref_img,
328         TEMP_FILE
329         )
330
331     try:
332         subprocess.check_output(command)
333     except subprocess.CalledProcessError as e:
334         if VERBOSE:
335             print_message(e.output.decode("utf-8"))
336
337     return not failed
338
339
340 def run_test(report, filepath):
341     testname = test_get_name(filepath)
342     spacer = "." * (32 - len(testname))
343     print_message(testname, 'SUCCESS', 'RUN')
344     time_start = time.time()
345     error = render_file(filepath)
346     status = "FAIL"
347     if not error:
348         if not verify_output(report, filepath):
349             error = "VERIFY"
350     time_end = time.time()
351     elapsed_ms = int((time_end - time_start) * 1000)
352     if not error:
353         print_message("{} ({} ms)" . format(testname, elapsed_ms),
354                       'SUCCESS', 'OK')
355     else:
356         if error == "NO_CYCLES":
357             print_message("Can't perform tests because Cycles failed to load!")
358             return error
359         elif error == "NO_START":
360             print_message('Can not perform tests because blender fails to start.',
361                   'Make sure INSTALL target was run.')
362             return error
363         elif error == 'VERIFY':
364             print_message("Rendered result is different from reference image")
365         else:
366             print_message("Unknown error %r" % error)
367         print_message("{} ({} ms)" . format(testname, elapsed_ms),
368                       'FAILURE', 'FAILED')
369     return error
370
371
372
373 def blend_list(path):
374     for dirpath, dirnames, filenames in os.walk(path):
375         for filename in filenames:
376             if filename.lower().endswith(".blend"):
377                 filepath = os.path.join(dirpath, filename)
378                 yield filepath
379
380 def run_all_tests(dirpath):
381     passed_tests = []
382     failed_tests = []
383     all_files = list(blend_list(dirpath))
384     all_files.sort()
385     report = Report(os.path.basename(dirpath))
386     print_message("Running {} tests from 1 test case." .
387                   format(len(all_files)),
388                   'SUCCESS', "==========")
389     time_start = time.time()
390     for filepath in all_files:
391         error = run_test(report, filepath)
392         testname = test_get_name(filepath)
393         if error:
394             if error == "NO_CYCLES":
395                 return False
396             elif error == "NO_START":
397                 return False
398             failed_tests.append(testname)
399         else:
400             passed_tests.append(testname)
401         report.add_test(filepath, error)
402     time_end = time.time()
403     elapsed_ms = int((time_end - time_start) * 1000)
404     print_message("")
405     print_message("{} tests from 1 test case ran. ({} ms total)" .
406                   format(len(all_files), elapsed_ms),
407                   'SUCCESS', "==========")
408     print_message("{} tests." .
409                   format(len(passed_tests)),
410                   'SUCCESS', 'PASSED')
411     if failed_tests:
412         print_message("{} tests, listed below:" .
413                      format(len(failed_tests)),
414                      'FAILURE', 'FAILED')
415         failed_tests.sort()
416         for test in failed_tests:
417             print_message("{}" . format(test), 'FAILURE', "FAILED")
418
419     report.output()
420     return not bool(failed_tests)
421
422
423 def create_argparse():
424     parser = argparse.ArgumentParser()
425     parser.add_argument("-blender", nargs="+")
426     parser.add_argument("-testdir", nargs=1)
427     parser.add_argument("-outdir", nargs=1)
428     parser.add_argument("-idiff", nargs=1)
429     return parser
430
431
432 def main():
433     parser = create_argparse()
434     args = parser.parse_args()
435
436     global COLORS
437     global BLENDER, TESTDIR, IDIFF, OUTDIR
438     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
439     global VERBOSE
440
441     if os.environ.get("CYCLESTEST_COLOR") is not None:
442         COLORS = COLORS_ANSI
443
444     BLENDER = args.blender[0]
445     TESTDIR = args.testdir[0]
446     IDIFF = args.idiff[0]
447     OUTDIR = args.outdir[0]
448
449     if not os.path.exists(OUTDIR):
450         os.makedirs(OUTDIR)
451
452     TEMP = tempfile.mkdtemp()
453     TEMP_FILE_MASK = os.path.join(TEMP, "test")
454     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
455
456     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
457
458     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
459
460     ok = run_all_tests(TESTDIR)
461
462     # Cleanup temp files and folders
463     if os.path.exists(TEMP_FILE):
464         os.remove(TEMP_FILE)
465     os.rmdir(TEMP)
466
467     sys.exit(not ok)
468
469
470 if __name__ == "__main__":
471     main()