Cycles tests: pass Blender custom arguments from CYCLESTEST_ARGS.
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import glob
6 import os
7 import pathlib
8 import shlex
9 import shutil
10 import subprocess
11 import sys
12 import time
13 import tempfile
14
15
16 class COLORS_ANSI:
17     RED = '\033[00;31m'
18     GREEN = '\033[00;32m'
19     ENDC = '\033[0m'
20
21
22 class COLORS_DUMMY:
23     RED = ''
24     GREEN = ''
25     ENDC = ''
26
27 COLORS = COLORS_DUMMY
28
29
30 def print_message(message, type=None, status=''):
31     if type == 'SUCCESS':
32         print(COLORS.GREEN, end="")
33     elif type == 'FAILURE':
34         print(COLORS.RED, end="")
35     status_text = ...
36     if status == 'RUN':
37         status_text = " RUN      "
38     elif status == 'OK':
39         status_text = "       OK "
40     elif status == 'PASSED':
41         status_text = "  PASSED  "
42     elif status == 'FAILED':
43         status_text = "  FAILED  "
44     else:
45         status_text = status
46     print("[{}]" . format(status_text), end="")
47     print(COLORS.ENDC, end="")
48     print(" {}" . format(message))
49     sys.stdout.flush()
50
51
52 def render_file(filepath):
53     dirname = os.path.dirname(filepath)
54     basedir = os.path.dirname(dirname)
55     subject = os.path.basename(dirname)
56
57     custom_args = os.getenv('CYCLESTEST_ARGS')
58     custom_args = shlex.split(custom_args) if custom_args else []
59
60     # OSL and GPU examples
61     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
62     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
63
64     if subject == 'opengl':
65         command = [
66             BLENDER,
67             "--window-geometry", "0", "0", "1", "1",
68             "-noaudio",
69             "--factory-startup",
70             "--enable-autoexec",
71             filepath,
72             "-E", "CYCLES"]
73         command += custom_args
74         command += [
75             "-o", TEMP_FILE_MASK,
76             "-F", "PNG",
77             '--python', os.path.join(basedir,
78                                      "util",
79                                      "render_opengl.py")]
80     else:
81         command = [
82             BLENDER,
83             "--background",
84             "-noaudio",
85             "--factory-startup",
86             "--enable-autoexec",
87             filepath,
88             "-E", "CYCLES"]
89         command += custom_args
90         command += [
91             "-o", TEMP_FILE_MASK,
92             "-F", "PNG",
93             "-f", "1"]
94     try:
95         output = subprocess.check_output(command)
96         if VERBOSE:
97             print(output.decode("utf-8"))
98         return None
99     except subprocess.CalledProcessError as e:
100         if os.path.exists(TEMP_FILE):
101             os.remove(TEMP_FILE)
102         if VERBOSE:
103             print(e.output.decode("utf-8"))
104         if b"Error: engine not found" in e.output:
105             return "NO_CYCLES"
106         elif b"blender probably wont start" in e.output:
107             return "NO_START"
108         return "CRASH"
109     except BaseException as e:
110         if os.path.exists(TEMP_FILE):
111             os.remove(TEMP_FILE)
112         if VERBOSE:
113             print(e)
114         return "CRASH"
115
116
117 def test_get_name(filepath):
118     filename = os.path.basename(filepath)
119     return os.path.splitext(filename)[0]
120
121 def test_get_images(filepath):
122     testname = test_get_name(filepath)
123     dirpath = os.path.dirname(filepath)
124
125     old_dirpath = os.path.join(dirpath, "reference_renders")
126     old_img = os.path.join(old_dirpath, testname + ".png")
127
128     ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref")
129     ref_img = os.path.join(ref_dirpath, testname + ".png")
130     if not os.path.exists(ref_dirpath):
131         os.makedirs(ref_dirpath)
132     if os.path.exists(old_img):
133         shutil.copy(old_img, ref_img)
134
135     new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
136     if not os.path.exists(new_dirpath):
137         os.makedirs(new_dirpath)
138     new_img = os.path.join(new_dirpath, testname + ".png")
139
140     diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
141     if not os.path.exists(diff_dirpath):
142         os.makedirs(diff_dirpath)
143     diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
144
145     return ref_img, new_img, diff_img
146
147
148 class Report:
149     def __init__(self, testname):
150         self.failed_tests = ""
151         self.passed_tests = ""
152         self.testname = testname
153
154     def output(self):
155         # write intermediate data for single test
156         outdir = os.path.join(OUTDIR, self.testname)
157         f = open(os.path.join(outdir, "failed.data"), "w")
158         f.write(self.failed_tests)
159         f.close()
160
161         f = open(os.path.join(outdir, "passed.data"), "w")
162         f.write(self.passed_tests)
163         f.close()
164
165         # gather intermediate data for all tests
166         failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
167         passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
168
169         failed_tests = ""
170         passed_tests = ""
171
172         for filename in failed_data:
173             failed_tests += open(os.path.join(OUTDIR, filename), "r").read()
174         for filename in passed_data:
175             passed_tests += open(os.path.join(OUTDIR, filename), "r").read()
176
177         # write html for all tests
178         self.html = """
179 <html>
180 <head>
181     <title>Cycles Test Report</title>
182     <style>
183         img {{ image-rendering: pixelated; width: 256px; background-color: #000; }}
184         img.render {{
185             background-color: #fff;
186             background-image:
187               -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
188               -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
189               -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
190               -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
191             background-image:
192               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
193               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
194               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
195               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
196
197             -moz-background-size:50px 50px;
198             background-size:50px 50px;
199             -webkit-background-size:50px 51px; /* override value for shitty webkit */
200
201             background-position:0 0, 25px 0, 25px -25px, 0px 25px;
202         }}
203         table td:first-child {{ width: 256px; }}
204     </style>
205     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
206 </head>
207 <body>
208     <div class="container">
209         <br/>
210         <h1>Cycles Test Report</h1>
211         <br/>
212         <table class="table table-striped">
213             <thead class="thead-default">
214                 <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
215             </thead>
216             {}{}
217         </table>
218         <br/>
219     </div>
220 </body>
221 </html>
222             """ . format(failed_tests, passed_tests)
223
224         filepath = os.path.join(OUTDIR, "report.html")
225         f = open(filepath, "w")
226         f.write(self.html)
227         f.close()
228
229         print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
230
231     def relative_url(self, filepath):
232         relpath = os.path.relpath(filepath, OUTDIR)
233         return pathlib.Path(relpath).as_posix()
234
235     def add_test(self, filepath, error):
236         name = test_get_name(filepath)
237         name = name.replace('_', ' ')
238
239         ref_img, new_img, diff_img = test_get_images(filepath)
240
241         status = error if error else ""
242         style = """ style="background-color: #f99;" """ if error else ""
243
244         new_url = self.relative_url(new_img)
245         ref_url = self.relative_url(ref_img)
246         diff_url = self.relative_url(diff_img)
247
248         test_html = """
249             <tr{}>
250                 <td><b>{}</b><br/>{}<br/>{}</td>
251                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
252                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
253                 <td><img src="{}"></td>
254             </tr>""" . format(style, name, self.testname, status,
255                               new_url, ref_url, new_url,
256                               ref_url, new_url, ref_url,
257                               diff_url)
258
259         if error:
260             self.failed_tests += test_html
261         else:
262             self.passed_tests += test_html
263
264
265 def verify_output(report, filepath):
266     ref_img, new_img, diff_img = test_get_images(filepath)
267
268     # copy new image
269     if os.path.exists(new_img):
270         os.remove(new_img)
271     if os.path.exists(TEMP_FILE):
272         shutil.copy(TEMP_FILE, new_img)
273
274
275     if not os.path.exists(ref_img):
276         return False
277
278     # diff test with threshold
279     command = (
280         IDIFF,
281         "-fail", "0.016",
282         "-failpercent", "1",
283         ref_img,
284         TEMP_FILE,
285         )
286     try:
287         subprocess.check_output(command)
288         failed = False
289     except subprocess.CalledProcessError as e:
290         if VERBOSE:
291             print_message(e.output.decode("utf-8"))
292         failed = e.returncode != 1
293
294     # generate diff image
295     command = (
296         IDIFF,
297         "-o", diff_img,
298         "-abs", "-scale", "16",
299         ref_img,
300         TEMP_FILE
301         )
302
303     try:
304         subprocess.check_output(command)
305     except subprocess.CalledProcessError as e:
306         if VERBOSE:
307             print_message(e.output.decode("utf-8"))
308
309     return not failed
310
311
312 def run_test(report, filepath):
313     testname = test_get_name(filepath)
314     spacer = "." * (32 - len(testname))
315     print_message(testname, 'SUCCESS', 'RUN')
316     time_start = time.time()
317     error = render_file(filepath)
318     status = "FAIL"
319     if not error:
320         if not verify_output(report, filepath):
321             error = "VERIFY"
322     time_end = time.time()
323     elapsed_ms = int((time_end - time_start) * 1000)
324     if not error:
325         print_message("{} ({} ms)" . format(testname, elapsed_ms),
326                       'SUCCESS', 'OK')
327     else:
328         if error == "NO_CYCLES":
329             print_message("Can't perform tests because Cycles failed to load!")
330             return error
331         elif error == "NO_START":
332             print_message('Can not perform tests because blender fails to start.',
333                   'Make sure INSTALL target was run.')
334             return error
335         elif error == 'VERIFY':
336             print_message("Rendered result is different from reference image")
337         else:
338             print_message("Unknown error %r" % error)
339         print_message("{} ({} ms)" . format(testname, elapsed_ms),
340                       'FAILURE', 'FAILED')
341     return error
342
343
344
345 def blend_list(path):
346     for dirpath, dirnames, filenames in os.walk(path):
347         for filename in filenames:
348             if filename.lower().endswith(".blend"):
349                 filepath = os.path.join(dirpath, filename)
350                 yield filepath
351
352 def run_all_tests(dirpath):
353     passed_tests = []
354     failed_tests = []
355     all_files = list(blend_list(dirpath))
356     all_files.sort()
357     report = Report(os.path.basename(dirpath))
358     print_message("Running {} tests from 1 test case." .
359                   format(len(all_files)),
360                   'SUCCESS', "==========")
361     time_start = time.time()
362     for filepath in all_files:
363         error = run_test(report, filepath)
364         testname = test_get_name(filepath)
365         if error:
366             if error == "NO_CYCLES":
367                 return False
368             elif error == "NO_START":
369                 return False
370             failed_tests.append(testname)
371         else:
372             passed_tests.append(testname)
373         report.add_test(filepath, error)
374     time_end = time.time()
375     elapsed_ms = int((time_end - time_start) * 1000)
376     print_message("")
377     print_message("{} tests from 1 test case ran. ({} ms total)" .
378                   format(len(all_files), elapsed_ms),
379                   'SUCCESS', "==========")
380     print_message("{} tests." .
381                   format(len(passed_tests)),
382                   'SUCCESS', 'PASSED')
383     if failed_tests:
384         print_message("{} tests, listed below:" .
385                      format(len(failed_tests)),
386                      'FAILURE', 'FAILED')
387         failed_tests.sort()
388         for test in failed_tests:
389             print_message("{}" . format(test), 'FAILURE', "FAILED")
390
391     report.output()
392     return not bool(failed_tests)
393
394
395 def create_argparse():
396     parser = argparse.ArgumentParser()
397     parser.add_argument("-blender", nargs="+")
398     parser.add_argument("-testdir", nargs=1)
399     parser.add_argument("-outdir", nargs=1)
400     parser.add_argument("-idiff", nargs=1)
401     return parser
402
403
404 def main():
405     parser = create_argparse()
406     args = parser.parse_args()
407
408     global COLORS
409     global BLENDER, TESTDIR, IDIFF, OUTDIR
410     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
411     global VERBOSE
412
413     if os.environ.get("CYCLESTEST_COLOR") is not None:
414         COLORS = COLORS_ANSI
415
416     BLENDER = args.blender[0]
417     TESTDIR = args.testdir[0]
418     IDIFF = args.idiff[0]
419     OUTDIR = args.outdir[0]
420
421     if not os.path.exists(OUTDIR):
422         os.makedirs(OUTDIR)
423
424     TEMP = tempfile.mkdtemp()
425     TEMP_FILE_MASK = os.path.join(TEMP, "test")
426     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
427
428     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
429
430     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
431
432     ok = run_all_tests(TESTDIR)
433
434     # Cleanup temp files and folders
435     if os.path.exists(TEMP_FILE):
436         os.remove(TEMP_FILE)
437     os.rmdir(TEMP)
438
439     sys.exit(not ok)
440
441
442 if __name__ == "__main__":
443     main()