Merge branch 'master' into blender2.8
[blender.git] / tests / python / modules / render_report.py
1 # Apache License, Version 2.0
2 #
3 # Compare renders or screenshots against reference versions and generate
4 # a HTML report showing the differences, for regression testing.
5
6 import glob
7 import os
8 import pathlib
9 import shutil
10 import subprocess
11 import sys
12 import time
13
14
15 class COLORS_ANSI:
16     RED = '\033[00;31m'
17     GREEN = '\033[00;32m'
18     ENDC = '\033[0m'
19
20
21 class COLORS_DUMMY:
22     RED = ''
23     GREEN = ''
24     ENDC = ''
25
26
27 COLORS = COLORS_DUMMY
28
29
30 def print_message(message, type=None, status=''):
31     if type == 'SUCCESS':
32         print(COLORS.GREEN, end="")
33     elif type == 'FAILURE':
34         print(COLORS.RED, end="")
35     status_text = ...
36     if status == 'RUN':
37         status_text = " RUN      "
38     elif status == 'OK':
39         status_text = "       OK "
40     elif status == 'PASSED':
41         status_text = "  PASSED  "
42     elif status == 'FAILED':
43         status_text = "  FAILED  "
44     else:
45         status_text = status
46     if status_text:
47         print("[{}]" . format(status_text), end="")
48     print(COLORS.ENDC, end="")
49     print(" {}" . format(message))
50     sys.stdout.flush()
51
52
53 def blend_list(dirpath):
54     for root, dirs, files in os.walk(dirpath):
55         for filename in files:
56             if filename.lower().endswith(".blend"):
57                 filepath = os.path.join(root, filename)
58                 yield filepath
59
60
61 def test_get_name(filepath):
62     filename = os.path.basename(filepath)
63     return os.path.splitext(filename)[0]
64
65
66 def test_get_images(output_dir, filepath, reference_dir):
67     testname = test_get_name(filepath)
68     dirpath = os.path.dirname(filepath)
69
70     old_dirpath = os.path.join(dirpath, reference_dir)
71     old_img = os.path.join(old_dirpath, testname + ".png")
72
73     ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref")
74     ref_img = os.path.join(ref_dirpath, testname + ".png")
75     if not os.path.exists(ref_dirpath):
76         os.makedirs(ref_dirpath)
77     if os.path.exists(old_img):
78         shutil.copy(old_img, ref_img)
79
80     new_dirpath = os.path.join(output_dir, os.path.basename(dirpath))
81     if not os.path.exists(new_dirpath):
82         os.makedirs(new_dirpath)
83     new_img = os.path.join(new_dirpath, testname + ".png")
84
85     diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff")
86     if not os.path.exists(diff_dirpath):
87         os.makedirs(diff_dirpath)
88     diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
89
90     return old_img, ref_img, new_img, diff_img
91
92
93 class Report:
94     __slots__ = (
95         'title',
96         'output_dir',
97         'reference_dir',
98         'idiff',
99         'pixelated',
100         'verbose',
101         'update',
102         'failed_tests',
103         'passed_tests',
104         'compare_tests',
105         'compare_engines'
106     )
107
108     def __init__(self, title, output_dir, idiff):
109         self.title = title
110         self.output_dir = output_dir
111         self.reference_dir = 'reference_renders'
112         self.idiff = idiff
113         self.compare_engines = None
114
115         self.pixelated = False
116         self.verbose = os.environ.get("BLENDER_VERBOSE") is not None
117         self.update = os.getenv('BLENDER_TEST_UPDATE') is not None
118
119         if os.environ.get("BLENDER_TEST_COLOR") is not None:
120             global COLORS, COLORS_ANSI
121             COLORS = COLORS_ANSI
122
123         self.failed_tests = ""
124         self.passed_tests = ""
125         self.compare_tests = ""
126
127         if not os.path.exists(output_dir):
128             os.makedirs(output_dir)
129
130     def set_pixelated(self, pixelated):
131         self.pixelated = pixelated
132
133     def set_reference_dir(self, reference_dir):
134         self.reference_dir = reference_dir
135
136     def set_compare_engines(self, engine, other_engine):
137         self.compare_engines = (engine, other_engine)
138
139     def run(self, dirpath, render_cb):
140         # Run tests and output report.
141         dirname = os.path.basename(dirpath)
142         ok = self._run_all_tests(dirname, dirpath, render_cb)
143         self._write_data(dirname)
144         self._write_html()
145         if self.compare_engines:
146             self._write_html(comparison=True)
147         return ok
148
149     def _write_data(self, dirname):
150         # Write intermediate data for single test.
151         outdir = os.path.join(self.output_dir, dirname)
152         if not os.path.exists(outdir):
153             os.makedirs(outdir)
154
155         filepath = os.path.join(outdir, "failed.data")
156         pathlib.Path(filepath).write_text(self.failed_tests)
157
158         filepath = os.path.join(outdir, "passed.data")
159         pathlib.Path(filepath).write_text(self.passed_tests)
160
161         if self.compare_engines:
162             filepath = os.path.join(outdir, "compare.data")
163             pathlib.Path(filepath).write_text(self.compare_tests)
164
165     def _write_html(self, comparison=False):
166         # Gather intermediate data for all tests.
167         if comparison:
168             failed_data = []
169             passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/compare.data")))
170         else:
171             failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data")))
172             passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data")))
173
174         failed_tests = ""
175         passed_tests = ""
176
177         for filename in failed_data:
178             filepath = os.path.join(self.output_dir, filename)
179             failed_tests += pathlib.Path(filepath).read_text()
180         for filename in passed_data:
181             filepath = os.path.join(self.output_dir, filename)
182             passed_tests += pathlib.Path(filepath).read_text()
183
184         tests_html = failed_tests + passed_tests
185
186         # Write html for all tests.
187         if self.pixelated:
188             image_rendering = 'pixelated'
189         else:
190             image_rendering = 'auto'
191
192         if len(failed_tests) > 0:
193             message = "<p>Run <tt>BLENDER_TEST_UPDATE=1 ctest</tt> to create or update reference images for failed tests.</p>"
194         else:
195             message = ""
196
197         if comparison:
198             title = "Render Test Compare"
199             columns_html = "<tr><th>Name</th><th>%s</th><th>%s</th>" % self.compare_engines
200         else:
201             title = self.title
202             columns_html = "<tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>"
203
204         html = """
205 <html>
206 <head>
207     <title>{title}</title>
208     <style>
209         img {{ image-rendering: {image_rendering}; width: 256px; background-color: #000; }}
210         img.render {{
211             background-color: #fff;
212             background-image:
213               -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
214               -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
215               -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
216               -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
217             background-image:
218               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
219               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
220               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
221               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
222
223             -moz-background-size:50px 50px;
224             background-size:50px 50px;
225             -webkit-background-size:50px 51px; /* override value for shitty webkit */
226
227             background-position:0 0, 25px 0, 25px -25px, 0px 25px;
228         }}
229         table td:first-child {{ width: 256px; }}
230     </style>
231     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
232 </head>
233 <body>
234     <div class="container">
235         <br/>
236         <h1>{title}</h1>
237         {message}
238         <br/>
239         <table class="table table-striped">
240             <thead class="thead-default">
241                 {columns_html}
242             </thead>
243             {tests_html}
244         </table>
245         <br/>
246     </div>
247 </body>
248 </html>
249             """ . format(title=title,
250                          message=message,
251                          image_rendering=image_rendering,
252                          tests_html=tests_html,
253                          columns_html=columns_html)
254
255         filename = "report.html" if not comparison else "compare.html"
256         filepath = os.path.join(self.output_dir, filename)
257         pathlib.Path(filepath).write_text(html)
258
259         print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
260
261     def _relative_url(self, filepath):
262         relpath = os.path.relpath(filepath, self.output_dir)
263         return pathlib.Path(relpath).as_posix()
264
265     def _write_test_html(self, testname, filepath, error):
266         name = test_get_name(filepath)
267         name = name.replace('_', ' ')
268
269         old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
270
271         status = error if error else ""
272         tr_style = """ style="background-color: #f99;" """ if error else ""
273
274         new_url = self._relative_url(new_img)
275         ref_url = self._relative_url(ref_img)
276         diff_url = self._relative_url(diff_img)
277
278         test_html = """
279             <tr{tr_style}>
280                 <td><b>{name}</b><br/>{testname}<br/>{status}</td>
281                 <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
282                 <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
283                 <td><img src="{diff_url}"></td>
284             </tr>""" . format(tr_style=tr_style,
285                               name=name,
286                               testname=testname,
287                               status=status,
288                               new_url=new_url,
289                               ref_url=ref_url,
290                               diff_url=diff_url)
291
292         if error:
293             self.failed_tests += test_html
294         else:
295             self.passed_tests += test_html
296
297         if self.compare_engines:
298             ref_url = os.path.join("..", self.compare_engines[1], new_url)
299
300             test_html = """
301                 <tr{tr_style}>
302                     <td><b>{name}</b><br/>{testname}<br/>{status}</td>
303                     <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
304                     <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
305                 </tr>""" . format(tr_style=tr_style,
306                                   name=name,
307                                   testname=testname,
308                                   status=status,
309                                   new_url=new_url,
310                                   ref_url=ref_url)
311
312             self.compare_tests += test_html
313
314     def _diff_output(self, filepath, tmp_filepath):
315         old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
316
317         # Create reference render directory.
318         old_dirpath = os.path.dirname(old_img)
319         if not os.path.exists(old_dirpath):
320             os.makedirs(old_dirpath)
321
322         # Copy temporary to new image.
323         if os.path.exists(new_img):
324             os.remove(new_img)
325         if os.path.exists(tmp_filepath):
326             shutil.copy(tmp_filepath, new_img)
327
328         if os.path.exists(ref_img):
329             # Diff images test with threshold.
330             command = (
331                 self.idiff,
332                 "-fail", "0.016",
333                 "-failpercent", "1",
334                 ref_img,
335                 tmp_filepath,
336             )
337             try:
338                 subprocess.check_output(command)
339                 failed = False
340             except subprocess.CalledProcessError as e:
341                 if self.verbose:
342                     print_message(e.output.decode("utf-8"))
343                 failed = e.returncode != 1
344         else:
345             if not self.update:
346                 return False
347
348             failed = True
349
350         if failed and self.update:
351             # Update reference image if requested.
352             shutil.copy(new_img, ref_img)
353             shutil.copy(new_img, old_img)
354             failed = False
355
356         # Generate diff image.
357         command = (
358             self.idiff,
359             "-o", diff_img,
360             "-abs", "-scale", "16",
361             ref_img,
362             tmp_filepath
363         )
364
365         try:
366             subprocess.check_output(command)
367         except subprocess.CalledProcessError as e:
368             if self.verbose:
369                 print_message(e.output.decode("utf-8"))
370
371         return not failed
372
373     def _run_test(self, filepath, render_cb):
374         testname = test_get_name(filepath)
375         print_message(testname, 'SUCCESS', 'RUN')
376         time_start = time.time()
377         tmp_filepath = os.path.join(self.output_dir, "tmp")
378
379         error = render_cb(filepath, tmp_filepath)
380         status = "FAIL"
381         if not error:
382             if not self._diff_output(filepath, tmp_filepath):
383                 error = "VERIFY"
384
385         if os.path.exists(tmp_filepath):
386             os.remove(tmp_filepath)
387
388         time_end = time.time()
389         elapsed_ms = int((time_end - time_start) * 1000)
390         if not error:
391             print_message("{} ({} ms)" . format(testname, elapsed_ms),
392                           'SUCCESS', 'OK')
393         else:
394             if error == "NO_ENGINE":
395                 print_message("Can't perform tests because the render engine failed to load!")
396                 return error
397             elif error == "NO_START":
398                 print_message('Can not perform tests because blender fails to start.',
399                               'Make sure INSTALL target was run.')
400                 return error
401             elif error == 'VERIFY':
402                 print_message("Rendered result is different from reference image")
403             else:
404                 print_message("Unknown error %r" % error)
405             print_message("{} ({} ms)" . format(testname, elapsed_ms),
406                           'FAILURE', 'FAILED')
407         return error
408
409     def _run_all_tests(self, dirname, dirpath, render_cb):
410         passed_tests = []
411         failed_tests = []
412         all_files = list(blend_list(dirpath))
413         all_files.sort()
414         print_message("Running {} tests from 1 test case." .
415                       format(len(all_files)),
416                       'SUCCESS', "==========")
417         time_start = time.time()
418         for filepath in all_files:
419             error = self._run_test(filepath, render_cb)
420             testname = test_get_name(filepath)
421             if error:
422                 if error == "NO_ENGINE":
423                     return False
424                 elif error == "NO_START":
425                     return False
426                 failed_tests.append(testname)
427             else:
428                 passed_tests.append(testname)
429             self._write_test_html(dirname, filepath, error)
430         time_end = time.time()
431         elapsed_ms = int((time_end - time_start) * 1000)
432         print_message("")
433         print_message("{} tests from 1 test case ran. ({} ms total)" .
434                       format(len(all_files), elapsed_ms),
435                       'SUCCESS', "==========")
436         print_message("{} tests." .
437                       format(len(passed_tests)),
438                       'SUCCESS', 'PASSED')
439         if failed_tests:
440             print_message("{} tests, listed below:" .
441                           format(len(failed_tests)),
442                           'FAILURE', 'FAILED')
443             failed_tests.sort()
444             for test in failed_tests:
445                 print_message("{}" . format(test), 'FAILURE', "FAILED")
446
447         return not bool(failed_tests)