77bee4953ba6593717f232134c5083bc52689462
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import glob
6 import os
7 import pathlib
8 import shutil
9 import subprocess
10 import sys
11 import time
12 import tempfile
13
14
15 class COLORS_ANSI:
16     RED = '\033[00;31m'
17     GREEN = '\033[00;32m'
18     ENDC = '\033[0m'
19
20
21 class COLORS_DUMMY:
22     RED = ''
23     GREEN = ''
24     ENDC = ''
25
26 COLORS = COLORS_DUMMY
27
28
29 def print_message(message, type=None, status=''):
30     if type == 'SUCCESS':
31         print(COLORS.GREEN, end="")
32     elif type == 'FAILURE':
33         print(COLORS.RED, end="")
34     status_text = ...
35     if status == 'RUN':
36         status_text = " RUN      "
37     elif status == 'OK':
38         status_text = "       OK "
39     elif status == 'PASSED':
40         status_text = "  PASSED  "
41     elif status == 'FAILED':
42         status_text = "  FAILED  "
43     else:
44         status_text = status
45     print("[{}]" . format(status_text), end="")
46     print(COLORS.ENDC, end="")
47     print(" {}" . format(message))
48     sys.stdout.flush()
49
50
51 def render_file(filepath):
52     dirname = os.path.dirname(filepath)
53     basedir = os.path.dirname(dirname)
54     subject = os.path.basename(dirname)
55     if subject == 'opengl':
56         command = (
57             BLENDER,
58             "--window-geometry", "0", "0", "1", "1",
59             "-noaudio",
60             "--factory-startup",
61             "--enable-autoexec",
62             filepath,
63             "-E", "CYCLES",
64             # Run with OSL enabled
65             # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True",
66             "-o", TEMP_FILE_MASK,
67             "-F", "PNG",
68             '--python', os.path.join(basedir,
69                                      "util",
70                                      "render_opengl.py")
71         )
72     else:
73         command = (
74             BLENDER,
75             "--background",
76             "-noaudio",
77             "--factory-startup",
78             "--enable-autoexec",
79             filepath,
80             "-E", "CYCLES",
81             # Run with OSL enabled
82             # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True",
83             "-o", TEMP_FILE_MASK,
84             "-F", "PNG",
85             "-f", "1",
86             )
87     try:
88         output = subprocess.check_output(command)
89         if VERBOSE:
90             print(output.decode("utf-8"))
91         return None
92     except subprocess.CalledProcessError as e:
93         if os.path.exists(TEMP_FILE):
94             os.remove(TEMP_FILE)
95         if VERBOSE:
96             print(e.output.decode("utf-8"))
97         if b"Error: engine not found" in e.output:
98             return "NO_CYCLES"
99         elif b"blender probably wont start" in e.output:
100             return "NO_START"
101         return "CRASH"
102     except BaseException as e:
103         if os.path.exists(TEMP_FILE):
104             os.remove(TEMP_FILE)
105         if VERBOSE:
106             print(e)
107         return "CRASH"
108
109
110 def test_get_name(filepath):
111     filename = os.path.basename(filepath)
112     return os.path.splitext(filename)[0]
113
114 def test_get_images(filepath):
115     testname = test_get_name(filepath)
116     dirpath = os.path.dirname(filepath)
117
118     old_dirpath = os.path.join(dirpath, "reference_renders")
119     old_img = os.path.join(old_dirpath, testname + ".png")
120
121     ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref")
122     ref_img = os.path.join(ref_dirpath, testname + ".png")
123     if not os.path.exists(ref_dirpath):
124         os.makedirs(ref_dirpath)
125     if os.path.exists(old_img):
126         shutil.copy(old_img, ref_img)
127
128     new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
129     if not os.path.exists(new_dirpath):
130         os.makedirs(new_dirpath)
131     new_img = os.path.join(new_dirpath, testname + ".png")
132
133     diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
134     if not os.path.exists(diff_dirpath):
135         os.makedirs(diff_dirpath)
136     diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
137
138     return ref_img, new_img, diff_img
139
140
141 class Report:
142     def __init__(self, testname):
143         self.failed_tests = ""
144         self.passed_tests = ""
145         self.testname = testname
146
147     def output(self):
148         # write intermediate data for single test
149         outdir = os.path.join(OUTDIR, self.testname)
150         f = open(os.path.join(outdir, "failed.data"), "w")
151         f.write(self.failed_tests)
152         f.close()
153
154         f = open(os.path.join(outdir, "passed.data"), "w")
155         f.write(self.passed_tests)
156         f.close()
157
158         # gather intermediate data for all tests
159         failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
160         passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
161
162         failed_tests = ""
163         passed_tests = ""
164
165         for filename in failed_data:
166             failed_tests += open(os.path.join(OUTDIR, filename), "r").read()
167         for filename in passed_data:
168             passed_tests += open(os.path.join(OUTDIR, filename), "r").read()
169
170         # write html for all tests
171         self.html = """
172 <html>
173 <head>
174     <title>Cycles Test Report</title>
175     <style>
176         img {{ image-rendering: pixelated; width: 256px; background-color: #000; }}
177         img.render {{
178             background-color: #fff;
179             background-image:
180               -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
181               -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
182               -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
183               -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
184             background-image:
185               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
186               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
187               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
188               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
189
190             -moz-background-size:50px 50px;
191             background-size:50px 50px;
192             -webkit-background-size:50px 51px; /* override value for shitty webkit */
193
194             background-position:0 0, 25px 0, 25px -25px, 0px 25px;
195         }}
196         table td:first-child {{ width: 256px; }}
197     </style>
198     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
199 </head>
200 <body>
201     <div class="container">
202         <br/>
203         <h1>Cycles Test Report</h1>
204         <br/>
205         <table class="table table-striped">
206             <thead class="thead-default">
207                 <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
208             </thead>
209             {}{}
210         </table>
211         <br/>
212     </div>
213 </body>
214 </html>
215             """ . format(failed_tests, passed_tests)
216
217         filepath = os.path.join(OUTDIR, "report.html")
218         f = open(filepath, "w")
219         f.write(self.html)
220         f.close()
221
222         print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
223
224     def relative_url(self, filepath):
225         relpath = os.path.relpath(filepath, OUTDIR)
226         return pathlib.Path(relpath).as_posix()
227
228     def add_test(self, filepath, error):
229         name = test_get_name(filepath)
230         name = name.replace('_', ' ')
231
232         ref_img, new_img, diff_img = test_get_images(filepath)
233
234         status = error if error else ""
235         style = """ style="background-color: #f99;" """ if error else ""
236
237         new_url = self.relative_url(new_img)
238         ref_url = self.relative_url(ref_img)
239         diff_url = self.relative_url(diff_img)
240
241         test_html = """
242             <tr{}>
243                 <td><b>{}</b><br/>{}<br/>{}</td>
244                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
245                 <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
246                 <td><img src="{}"></td>
247             </tr>""" . format(style, name, self.testname, status,
248                               new_url, ref_url, new_url,
249                               ref_url, new_url, ref_url,
250                               diff_url)
251
252         if error:
253             self.failed_tests += test_html
254         else:
255             self.passed_tests += test_html
256
257
258 def verify_output(report, filepath):
259     ref_img, new_img, diff_img = test_get_images(filepath)
260
261     # copy new image
262     if os.path.exists(new_img):
263         os.remove(new_img)
264     if os.path.exists(TEMP_FILE):
265         shutil.copy(TEMP_FILE, new_img)
266
267
268     if not os.path.exists(ref_img):
269         return False
270
271     # diff test with threshold
272     command = (
273         IDIFF,
274         "-fail", "0.016",
275         "-failpercent", "1",
276         ref_img,
277         TEMP_FILE,
278         )
279     try:
280         subprocess.check_output(command)
281         failed = False
282     except subprocess.CalledProcessError as e:
283         if VERBOSE:
284             print_message(e.output.decode("utf-8"))
285         failed = e.returncode != 1
286
287     # generate diff image
288     command = (
289         IDIFF,
290         "-o", diff_img,
291         "-abs", "-scale", "16",
292         ref_img,
293         TEMP_FILE
294         )
295
296     try:
297         subprocess.check_output(command)
298     except subprocess.CalledProcessError as e:
299         if VERBOSE:
300             print_message(e.output.decode("utf-8"))
301
302     return not failed
303
304
305 def run_test(report, filepath):
306     testname = test_get_name(filepath)
307     spacer = "." * (32 - len(testname))
308     print_message(testname, 'SUCCESS', 'RUN')
309     time_start = time.time()
310     error = render_file(filepath)
311     status = "FAIL"
312     if not error:
313         if not verify_output(report, filepath):
314             error = "VERIFY"
315     time_end = time.time()
316     elapsed_ms = int((time_end - time_start) * 1000)
317     if not error:
318         print_message("{} ({} ms)" . format(testname, elapsed_ms),
319                       'SUCCESS', 'OK')
320     else:
321         if error == "NO_CYCLES":
322             print_message("Can't perform tests because Cycles failed to load!")
323             return error
324         elif error == "NO_START":
325             print_message('Can not perform tests because blender fails to start.',
326                   'Make sure INSTALL target was run.')
327             return error
328         elif error == 'VERIFY':
329             print_message("Rendered result is different from reference image")
330         else:
331             print_message("Unknown error %r" % error)
332         print_message("{} ({} ms)" . format(testname, elapsed_ms),
333                       'FAILURE', 'FAILED')
334     return error
335
336
337
338 def blend_list(path):
339     for dirpath, dirnames, filenames in os.walk(path):
340         for filename in filenames:
341             if filename.lower().endswith(".blend"):
342                 filepath = os.path.join(dirpath, filename)
343                 yield filepath
344
345 def run_all_tests(dirpath):
346     passed_tests = []
347     failed_tests = []
348     all_files = list(blend_list(dirpath))
349     all_files.sort()
350     report = Report(os.path.basename(dirpath))
351     print_message("Running {} tests from 1 test case." .
352                   format(len(all_files)),
353                   'SUCCESS', "==========")
354     time_start = time.time()
355     for filepath in all_files:
356         error = run_test(report, filepath)
357         testname = test_get_name(filepath)
358         if error:
359             if error == "NO_CYCLES":
360                 return False
361             elif error == "NO_START":
362                 return False
363             failed_tests.append(testname)
364         else:
365             passed_tests.append(testname)
366         report.add_test(filepath, error)
367     time_end = time.time()
368     elapsed_ms = int((time_end - time_start) * 1000)
369     print_message("")
370     print_message("{} tests from 1 test case ran. ({} ms total)" .
371                   format(len(all_files), elapsed_ms),
372                   'SUCCESS', "==========")
373     print_message("{} tests." .
374                   format(len(passed_tests)),
375                   'SUCCESS', 'PASSED')
376     if failed_tests:
377         print_message("{} tests, listed below:" .
378                      format(len(failed_tests)),
379                      'FAILURE', 'FAILED')
380         failed_tests.sort()
381         for test in failed_tests:
382             print_message("{}" . format(test), 'FAILURE', "FAILED")
383
384     report.output()
385     return not bool(failed_tests)
386
387
388 def create_argparse():
389     parser = argparse.ArgumentParser()
390     parser.add_argument("-blender", nargs="+")
391     parser.add_argument("-testdir", nargs=1)
392     parser.add_argument("-outdir", nargs=1)
393     parser.add_argument("-idiff", nargs=1)
394     return parser
395
396
397 def main():
398     parser = create_argparse()
399     args = parser.parse_args()
400
401     global COLORS
402     global BLENDER, TESTDIR, IDIFF, OUTDIR
403     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
404     global VERBOSE
405
406     if os.environ.get("CYCLESTEST_COLOR") is not None:
407         COLORS = COLORS_ANSI
408
409     BLENDER = args.blender[0]
410     TESTDIR = args.testdir[0]
411     IDIFF = args.idiff[0]
412     OUTDIR = args.outdir[0]
413
414     if not os.path.exists(OUTDIR):
415         os.makedirs(OUTDIR)
416
417     TEMP = tempfile.mkdtemp()
418     TEMP_FILE_MASK = os.path.join(TEMP, "test")
419     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
420
421     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
422
423     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
424
425     ok = run_all_tests(TESTDIR)
426
427     # Cleanup temp files and folders
428     if os.path.exists(TEMP_FILE):
429         os.remove(TEMP_FILE)
430     os.rmdir(TEMP)
431
432     sys.exit(not ok)
433
434
435 if __name__ == "__main__":
436     main()