15826f974008d39abca041ac2afdf0c0bc45b7b3
[blender.git] / tests / python / modules / render_report.py
1 # Apache License, Version 2.0
2 #
3 # Compare renders or screenshots against reference versions and generate
4 # a HTML report showing the differences, for regression testing.
5
6 import glob
7 import os
8 import pathlib
9 import shutil
10 import subprocess
11 import sys
12 import time
13
14 from . import global_report
15
16
17 class COLORS_ANSI:
18     RED = '\033[00;31m'
19     GREEN = '\033[00;32m'
20     ENDC = '\033[0m'
21
22
23 class COLORS_DUMMY:
24     RED = ''
25     GREEN = ''
26     ENDC = ''
27
28
29 COLORS = COLORS_DUMMY
30
31
32 def print_message(message, type=None, status=''):
33     if type == 'SUCCESS':
34         print(COLORS.GREEN, end="")
35     elif type == 'FAILURE':
36         print(COLORS.RED, end="")
37     status_text = ...
38     if status == 'RUN':
39         status_text = " RUN      "
40     elif status == 'OK':
41         status_text = "       OK "
42     elif status == 'PASSED':
43         status_text = "  PASSED  "
44     elif status == 'FAILED':
45         status_text = "  FAILED  "
46     else:
47         status_text = status
48     if status_text:
49         print("[{}]" . format(status_text), end="")
50     print(COLORS.ENDC, end="")
51     print(" {}" . format(message))
52     sys.stdout.flush()
53
54
55 def blend_list(dirpath):
56     for root, dirs, files in os.walk(dirpath):
57         for filename in files:
58             if filename.lower().endswith(".blend"):
59                 filepath = os.path.join(root, filename)
60                 yield filepath
61
62
63 def test_get_name(filepath):
64     filename = os.path.basename(filepath)
65     return os.path.splitext(filename)[0]
66
67
68 def test_get_images(output_dir, filepath, reference_dir):
69     testname = test_get_name(filepath)
70     dirpath = os.path.dirname(filepath)
71
72     old_dirpath = os.path.join(dirpath, reference_dir)
73     old_img = os.path.join(old_dirpath, testname + ".png")
74
75     ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref")
76     ref_img = os.path.join(ref_dirpath, testname + ".png")
77     os.makedirs(ref_dirpath, exist_ok=True)
78     if os.path.exists(old_img):
79         shutil.copy(old_img, ref_img)
80
81     new_dirpath = os.path.join(output_dir, os.path.basename(dirpath))
82     os.makedirs(new_dirpath, exist_ok=True)
83     new_img = os.path.join(new_dirpath, testname + ".png")
84
85     diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff")
86     os.makedirs(diff_dirpath, exist_ok=True)
87     diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
88
89     return old_img, ref_img, new_img, diff_img
90
91
92 class Report:
93     __slots__ = (
94         'title',
95         'output_dir',
96         'reference_dir',
97         'idiff',
98         'pixelated',
99         'verbose',
100         'update',
101         'failed_tests',
102         'passed_tests',
103         'compare_tests',
104         'compare_engines'
105     )
106
107     def __init__(self, title, output_dir, idiff):
108         self.title = title
109         self.output_dir = output_dir
110         self.reference_dir = 'reference_renders'
111         self.idiff = idiff
112         self.compare_engines = None
113
114         self.pixelated = False
115         self.verbose = os.environ.get("BLENDER_VERBOSE") is not None
116         self.update = os.getenv('BLENDER_TEST_UPDATE') is not None
117
118         if os.environ.get("BLENDER_TEST_COLOR") is not None:
119             global COLORS, COLORS_ANSI
120             COLORS = COLORS_ANSI
121
122         self.failed_tests = ""
123         self.passed_tests = ""
124         self.compare_tests = ""
125
126         os.makedirs(output_dir, exist_ok=True)
127
128     def set_pixelated(self, pixelated):
129         self.pixelated = pixelated
130
131     def set_reference_dir(self, reference_dir):
132         self.reference_dir = reference_dir
133
134     def set_compare_engines(self, engine, other_engine):
135         self.compare_engines = (engine, other_engine)
136
137     def run(self, dirpath, blender, arguments_cb, batch=False):
138         # Run tests and output report.
139         dirname = os.path.basename(dirpath)
140         ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch)
141         self._write_data(dirname)
142         self._write_html()
143         if self.compare_engines:
144             self._write_html(comparison=True)
145         return ok
146
147     def _write_data(self, dirname):
148         # Write intermediate data for single test.
149         outdir = os.path.join(self.output_dir, dirname)
150         os.makedirs(outdir, exist_ok=True)
151
152         filepath = os.path.join(outdir, "failed.data")
153         pathlib.Path(filepath).write_text(self.failed_tests)
154
155         filepath = os.path.join(outdir, "passed.data")
156         pathlib.Path(filepath).write_text(self.passed_tests)
157
158         if self.compare_engines:
159             filepath = os.path.join(outdir, "compare.data")
160             pathlib.Path(filepath).write_text(self.compare_tests)
161
162     def _navigation_item(self, title, href, active):
163         if active:
164             return """<li class="breadcrumb-item active" aria-current="page">%s</li>""" % title
165         else:
166             return """<li class="breadcrumb-item"><a href="%s">%s</a></li>""" % (href, title)
167
168     def _navigation_html(self, comparison):
169         html = """<nav aria-label="breadcrumb"><ol class="breadcrumb">"""
170         html += self._navigation_item("Test Reports", "../report.html", False)
171         html += self._navigation_item(self.title, "report.html", not comparison)
172         if self.compare_engines:
173             compare_title = "Compare with %s" % self.compare_engines[1].capitalize()
174             html += self._navigation_item(compare_title, "compare.html", comparison)
175         html += """</ol></nav>"""
176
177         return html
178
179     def _write_html(self, comparison=False):
180         # Gather intermediate data for all tests.
181         if comparison:
182             failed_data = []
183             passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/compare.data")))
184         else:
185             failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data")))
186             passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data")))
187
188         failed_tests = ""
189         passed_tests = ""
190
191         for filename in failed_data:
192             filepath = os.path.join(self.output_dir, filename)
193             failed_tests += pathlib.Path(filepath).read_text()
194         for filename in passed_data:
195             filepath = os.path.join(self.output_dir, filename)
196             passed_tests += pathlib.Path(filepath).read_text()
197
198         tests_html = failed_tests + passed_tests
199
200         # Write html for all tests.
201         if self.pixelated:
202             image_rendering = 'pixelated'
203         else:
204             image_rendering = 'auto'
205
206         # Navigation
207         menu = self._navigation_html(comparison)
208
209         failed = len(failed_tests) > 0
210         if failed:
211             message = """<div class="alert alert-danger" role="alert">"""
212             message += """Run this command to update reference images for failed tests, or create images for new tests:<br>"""
213             message += """<tt>BLENDER_TEST_UPDATE=1 ctest -R %s</tt>""" % self.title.lower()
214             message += """</div>"""
215         else:
216             message = ""
217
218         if comparison:
219             title = self.title + " Test Compare"
220             engine_self = self.compare_engines[0].capitalize()
221             engine_other = self.compare_engines[1].capitalize()
222             columns_html = "<tr><th>Name</th><th>%s</th><th>%s</th>" % (engine_self, engine_other)
223         else:
224             title = self.title + " Test Report"
225             columns_html = "<tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>"
226
227         html = """
228 <html>
229 <head>
230     <title>{title}</title>
231     <style>
232         img {{ image-rendering: {image_rendering}; width: 256px; background-color: #000; }}
233         img.render {{
234             background-color: #fff;
235             background-image:
236               -moz-linear-gradient(45deg, #eee 25%, transparent 25%),
237               -moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
238               -moz-linear-gradient(45deg, transparent 75%, #eee 75%),
239               -moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
240             background-image:
241               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
242               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
243               -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
244               -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
245
246             -moz-background-size:50px 50px;
247             background-size:50px 50px;
248             -webkit-background-size:50px 51px; /* override value for shitty webkit */
249
250             background-position:0 0, 25px 0, 25px -25px, 0px 25px;
251         }}
252         table td:first-child {{ width: 256px; }}
253     </style>
254     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
255 </head>
256 <body>
257     <div class="container">
258         <br/>
259         <h1>{title}</h1>
260         {menu}
261         {message}
262         <table class="table table-striped">
263             <thead class="thead-dark">
264                 {columns_html}
265             </thead>
266             {tests_html}
267         </table>
268         <br/>
269     </div>
270 </body>
271 </html>
272             """ . format(title=title,
273                          menu=menu,
274                          message=message,
275                          image_rendering=image_rendering,
276                          tests_html=tests_html,
277                          columns_html=columns_html)
278
279         filename = "report.html" if not comparison else "compare.html"
280         filepath = os.path.join(self.output_dir, filename)
281         pathlib.Path(filepath).write_text(html)
282
283         print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
284
285         # Update global report
286         if not comparison:
287             global_output_dir = os.path.dirname(self.output_dir)
288             global_failed = failed if not comparison else None
289             global_report.add(global_output_dir, "Render", self.title, filepath, global_failed)
290
291     def _relative_url(self, filepath):
292         relpath = os.path.relpath(filepath, self.output_dir)
293         return pathlib.Path(relpath).as_posix()
294
295     def _write_test_html(self, testname, filepath, error):
296         name = test_get_name(filepath)
297         name = name.replace('_', ' ')
298
299         old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
300
301         status = error if error else ""
302         tr_style = """ class="table-danger" """ if error else ""
303
304         new_url = self._relative_url(new_img)
305         ref_url = self._relative_url(ref_img)
306         diff_url = self._relative_url(diff_img)
307
308         test_html = """
309             <tr{tr_style}>
310                 <td><b>{name}</b><br/>{testname}<br/>{status}</td>
311                 <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
312                 <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
313                 <td><img src="{diff_url}"></td>
314             </tr>""" . format(tr_style=tr_style,
315                               name=name,
316                               testname=testname,
317                               status=status,
318                               new_url=new_url,
319                               ref_url=ref_url,
320                               diff_url=diff_url)
321
322         if error:
323             self.failed_tests += test_html
324         else:
325             self.passed_tests += test_html
326
327         if self.compare_engines:
328             ref_url = os.path.join("..", self.compare_engines[1], new_url)
329
330             test_html = """
331                 <tr{tr_style}>
332                     <td><b>{name}</b><br/>{testname}<br/>{status}</td>
333                     <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td>
334                     <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td>
335                 </tr>""" . format(tr_style=tr_style,
336                                   name=name,
337                                   testname=testname,
338                                   status=status,
339                                   new_url=new_url,
340                                   ref_url=ref_url)
341
342             self.compare_tests += test_html
343
344     def _diff_output(self, filepath, tmp_filepath):
345         old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir)
346
347         # Create reference render directory.
348         old_dirpath = os.path.dirname(old_img)
349         os.makedirs(old_dirpath, exist_ok=True)
350
351         # Copy temporary to new image.
352         if os.path.exists(new_img):
353             os.remove(new_img)
354         if os.path.exists(tmp_filepath):
355             shutil.copy(tmp_filepath, new_img)
356
357         if os.path.exists(ref_img):
358             # Diff images test with threshold.
359             command = (
360                 self.idiff,
361                 "-fail", "0.016",
362                 "-failpercent", "1",
363                 ref_img,
364                 tmp_filepath,
365             )
366             try:
367                 subprocess.check_output(command)
368                 failed = False
369             except subprocess.CalledProcessError as e:
370                 if self.verbose:
371                     print_message(e.output.decode("utf-8"))
372                 failed = e.returncode != 1
373         else:
374             if not self.update:
375                 return False
376
377             failed = True
378
379         if failed and self.update:
380             # Update reference image if requested.
381             shutil.copy(new_img, ref_img)
382             shutil.copy(new_img, old_img)
383             failed = False
384
385         # Generate diff image.
386         command = (
387             self.idiff,
388             "-o", diff_img,
389             "-abs", "-scale", "16",
390             ref_img,
391             tmp_filepath
392         )
393
394         try:
395             subprocess.check_output(command)
396         except subprocess.CalledProcessError as e:
397             if self.verbose:
398                 print_message(e.output.decode("utf-8"))
399
400         return not failed
401
402     def _run_tests(self, filepaths, blender, arguments_cb, batch):
403         # Run multiple tests in a single Blender process since startup can be
404         # a significant factor. In case of crashes, re-run the remaining tests.
405         verbose = os.environ.get("BLENDER_VERBOSE") is not None
406
407         remaining_filepaths = filepaths[:]
408         errors = []
409
410         while len(remaining_filepaths) > 0:
411             command = [blender]
412             output_filepaths = []
413
414             # Construct output filepaths and command to run
415             for filepath in remaining_filepaths:
416                 testname = test_get_name(filepath)
417                 print_message(testname, 'SUCCESS', 'RUN')
418
419                 base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname)
420                 output_filepath = base_output_filepath + '0001.png'
421                 output_filepaths.append(output_filepath)
422
423                 if os.path.exists(output_filepath):
424                     os.remove(output_filepath)
425
426                 command.extend(arguments_cb(filepath, base_output_filepath))
427
428                 # Only chain multiple commands for batch
429                 if not batch:
430                     break
431
432             # Run process
433             crash = False
434             try:
435                 output = subprocess.check_output(command)
436             except subprocess.CalledProcessError as e:
437                 crash = True
438             except BaseException as e:
439                 crash = True
440
441             if verbose:
442                 print(" ".join(command))
443                 print(output.decode("utf-8"))
444
445             # Detect missing filepaths and consider those errors
446             for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths):
447                 remaining_filepaths.pop(0)
448
449                 if crash:
450                     # In case of crash, stop after missing files and re-render remaing
451                     if not os.path.exists(output_filepath):
452                         errors.append("CRASH")
453                         print_message("Crash running Blender")
454                         print_message(testname, 'FAILURE', 'FAILED')
455                         break
456
457                 testname = test_get_name(filepath)
458
459                 if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0:
460                     errors.append("NO OUTPUT")
461                     print_message("No render result file found")
462                     print_message(testname, 'FAILURE', 'FAILED')
463                 elif not self._diff_output(filepath, output_filepath):
464                     errors.append("VERIFY")
465                     print_message("Render result is different from reference image")
466                     print_message(testname, 'FAILURE', 'FAILED')
467                 else:
468                     errors.append(None)
469                     print_message(testname, 'SUCCESS', 'OK')
470
471                 if os.path.exists(output_filepath):
472                     os.remove(output_filepath)
473
474         return errors
475
476     def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch):
477         passed_tests = []
478         failed_tests = []
479         all_files = list(blend_list(dirpath))
480         all_files.sort()
481         print_message("Running {} tests from 1 test case." .
482                       format(len(all_files)),
483                       'SUCCESS', "==========")
484         time_start = time.time()
485         errors = self._run_tests(all_files, blender, arguments_cb, batch)
486         for filepath, error in zip(all_files, errors):
487             testname = test_get_name(filepath)
488             if error:
489                 if error == "NO_ENGINE":
490                     return False
491                 elif error == "NO_START":
492                     return False
493                 failed_tests.append(testname)
494             else:
495                 passed_tests.append(testname)
496             self._write_test_html(dirname, filepath, error)
497         time_end = time.time()
498         elapsed_ms = int((time_end - time_start) * 1000)
499         print_message("")
500         print_message("{} tests from 1 test case ran. ({} ms total)" .
501                       format(len(all_files), elapsed_ms),
502                       'SUCCESS', "==========")
503         print_message("{} tests." .
504                       format(len(passed_tests)),
505                       'SUCCESS', 'PASSED')
506         if failed_tests:
507             print_message("{} tests, listed below:" .
508                           format(len(failed_tests)),
509                           'FAILURE', 'FAILED')
510             failed_tests.sort()
511             for test in failed_tests:
512                 print_message("{}" . format(test), 'FAILURE', "FAILED")
513
514         return not bool(failed_tests)