Cycles: add HTML report to inspect failed test images.
authorBrecht Van Lommel <brechtvanlommel@gmail.com>
Thu, 3 Aug 2017 14:41:50 +0000 (16:41 +0200)
committerBrecht Van Lommel <brechtvanlommel@gmail.com>
Mon, 7 Aug 2017 12:01:24 +0000 (14:01 +0200)
Shows new, reference and diff renders, with mouse hover to flip between
new and ref for easy comparison. This generates a report.html in
build_dir/tests/cycles, stored along with the new and diff images.

Differential Revision: https://developer.blender.org/D2770

tests/python/CMakeLists.txt
tests/python/cycles_render_tests.py

index f2fea48..4ba89c2 100644 (file)
@@ -518,6 +518,7 @@ if(WITH_CYCLES)
                                        -blender "$<TARGET_FILE:blender>"
                                        -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}"
                                        -idiff "${OPENIMAGEIO_IDIFF}"
+                                       -outdir "${TEST_OUT_DIR}/cycles"
                                )
                        else()
                                add_test(
@@ -526,6 +527,7 @@ if(WITH_CYCLES)
                                        -blender "$<TARGET_FILE:blender>"
                                        -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}"
                                        -idiff "${OPENIMAGEIO_IDIFF}"
+                                       -outdir "${TEST_OUT_DIR}/cycles"
                                )
                        endif()
                endmacro()
index a030cc5..ea84f27 100755 (executable)
@@ -2,7 +2,9 @@
 # Apache License, Version 2.0
 
 import argparse
+import glob
 import os
+import pathlib
 import shutil
 import subprocess
 import sys
@@ -24,7 +26,7 @@ class COLORS_DUMMY:
 COLORS = COLORS_DUMMY
 
 
-def printMessage(type, status, message):
+def print_message(message, type=None, status=''):
     if type == 'SUCCESS':
         print(COLORS.GREEN, end="")
     elif type == 'FAILURE':
@@ -109,20 +111,126 @@ def test_get_name(filepath):
     filename = os.path.basename(filepath)
     return os.path.splitext(filename)[0]
 
-
-def verify_output(filepath):
+def test_get_images(filepath):
     testname = test_get_name(filepath)
     dirpath = os.path.dirname(filepath)
-    reference_dirpath = os.path.join(dirpath, "reference_renders")
-    reference_image = os.path.join(reference_dirpath, testname + ".png")
-    failed_image = os.path.join(reference_dirpath, testname + ".fail.png")
-    if not os.path.exists(reference_image):
+    ref_dirpath = os.path.join(dirpath, "reference_renders")
+    ref_img = os.path.join(ref_dirpath, testname + ".png")
+    new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
+    if not os.path.exists(new_dirpath):
+        os.makedirs(new_dirpath)
+    new_img = os.path.join(new_dirpath, testname + ".png")
+    diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
+    if not os.path.exists(diff_dirpath):
+        os.makedirs(diff_dirpath)
+    diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
+    return ref_img, new_img, diff_img
+
+
+class Report:
+    def __init__(self, testname):
+        self.failed_tests = ""
+        self.passed_tests = ""
+        self.testname = testname
+
+    def output(self):
+        # write intermediate data for single test
+        outdir = os.path.join(OUTDIR, self.testname)
+        f = open(os.path.join(outdir, "failed.data"), "w")
+        f.write(self.failed_tests)
+        f.close()
+
+        f = open(os.path.join(outdir, "passed.data"), "w")
+        f.write(self.passed_tests)
+        f.close()
+
+        # gather intermediate data for all tests
+        failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
+        passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
+
+        failed_tests = ""
+        passed_tests = ""
+
+        for filename in failed_data:
+            failed_tests += open(os.path.join(OUTDIR, filename), "r").read()
+        for filename in passed_data:
+            passed_tests += open(os.path.join(OUTDIR, filename), "r").read()
+
+        # write html for all tests
+        self.html = """
+<html>
+<head>
+    <title>Cycles Test Report</title>
+    <style>
+        img {{ image-rendering: pixelated; width: 256; background-color: #000; }}
+        table td:first-child {{ width: 100%; }}
+    </style>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
+</head>
+<body>
+    <div class="container">
+        <br/>
+        <h1>Cycles Test Report</h1>
+        <br/>
+        <table class="table table-striped">
+            <thead class="thead-default">
+                <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
+            </thead>
+            {}{}
+        </table>
+        <br/>
+    </div>
+</body>
+</html>
+            """ . format(failed_tests, passed_tests)
+
+        filepath = os.path.join(OUTDIR, "report.html")
+        f = open(filepath, "w")
+        f.write(self.html)
+        f.close()
+
+        print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
+
+    def add_test(self, filepath, error):
+        name = test_get_name(filepath)
+
+        ref_img, new_img, diff_img = test_get_images(filepath)
+
+        status = error if error else ""
+        style = """ style="background-color: #f99;" """ if error else ""
+
+        new_url = pathlib.Path(new_img).as_uri()
+        ref_url = pathlib.Path(ref_img).as_uri()
+        diff_url = pathlib.Path(diff_img).as_uri()
+
+        test_html = """
+            <tr{}>
+                <td><b>{}</b><br/>{}<br/>{}</td>
+                <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td>
+                <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';"></td>
+                <td><img src="{}"></td>
+            </tr>""" . format(style, name, self.testname, status,
+                              new_url, ref_url, new_url,
+                              ref_url, new_url, ref_url,
+                              diff_url)
+
+        if error:
+            self.failed_tests += test_html
+        else:
+            self.passed_tests += test_html
+
+
+def verify_output(report, filepath):
+    ref_img, new_img, diff_img = test_get_images(filepath)
+    if not os.path.exists(ref_img):
         return False
+
+    # diff test with threshold
     command = (
         IDIFF,
-        "-fail", "0.015",
+        "-fail", "0.016",
         "-failpercent", "1",
-        reference_image,
+        ref_img,
         TEMP_FILE,
         )
     try:
@@ -130,47 +238,66 @@ def verify_output(filepath):
         failed = False
     except subprocess.CalledProcessError as e:
         if VERBOSE:
-            print(e.output.decode("utf-8"))
+            print_message(e.output.decode("utf-8"))
         failed = e.returncode != 1
-    if failed:
-        shutil.copy(TEMP_FILE, failed_image)
-    elif os.path.exists(failed_image):
-        os.remove(failed_image)
+
+    # generate diff image
+    command = (
+        IDIFF,
+        "-o", diff_img,
+        "-abs", "-scale", "16",
+        ref_img,
+        TEMP_FILE
+        )
+
+    try:
+        subprocess.check_output(command)
+    except subprocess.CalledProcessError as e:
+        if VERBOSE:
+            print_message(e.output.decode("utf-8"))
+
+    # copy new image
+    if os.path.exists(new_img):
+        os.remove(new_img)
+    if os.path.exists(TEMP_FILE):
+        shutil.copy(TEMP_FILE, new_img)
+
     return not failed
 
 
-def run_test(filepath):
+def run_test(report, filepath):
     testname = test_get_name(filepath)
     spacer = "." * (32 - len(testname))
-    printMessage('SUCCESS', 'RUN', testname)
+    print_message(testname, 'SUCCESS', 'RUN')
     time_start = time.time()
     error = render_file(filepath)
     status = "FAIL"
     if not error:
-        if not verify_output(filepath):
+        if not verify_output(report, filepath):
             error = "VERIFY"
     time_end = time.time()
     elapsed_ms = int((time_end - time_start) * 1000)
     if not error:
-        printMessage('SUCCESS', 'OK', "{} ({} ms)" .
-                     format(testname, elapsed_ms))
+        print_message("{} ({} ms)" . format(testname, elapsed_ms),
+                      'SUCCESS', 'OK')
     else:
         if error == "NO_CYCLES":
-            print("Can't perform tests because Cycles failed to load!")
-            return False
+            print_message("Can't perform tests because Cycles failed to load!")
+            return error
         elif error == "NO_START":
-            print('Can not perform tests because blender fails to start.',
+            print_message('Can not perform tests because blender fails to start.',
                   'Make sure INSTALL target was run.')
-            return False
+            return error
         elif error == 'VERIFY':
-            print("Rendered result is different from reference image")
+            print_message("Rendered result is different from reference image")
         else:
-            print("Unknown error %r" % error)
-        printMessage('FAILURE', 'FAILED', "{} ({} ms)" .
-                     format(testname, elapsed_ms))
+            print_message("Unknown error %r" % error)
+        print_message("{} ({} ms)" . format(testname, elapsed_ms),
+                      'FAILURE', 'FAILED')
     return error
 
 
+
 def blend_list(path):
     for dirpath, dirnames, filenames in os.walk(path):
         for filename in filenames:
@@ -178,17 +305,18 @@ def blend_list(path):
                 filepath = os.path.join(dirpath, filename)
                 yield filepath
 
-
 def run_all_tests(dirpath):
     passed_tests = []
     failed_tests = []
     all_files = list(blend_list(dirpath))
     all_files.sort()
-    printMessage('SUCCESS', "==========",
-                 "Running {} tests from 1 test case." . format(len(all_files)))
+    report = Report(os.path.basename(dirpath))
+    print_message("Running {} tests from 1 test case." .
+                  format(len(all_files)),
+                  'SUCCESS', "==========")
     time_start = time.time()
     for filepath in all_files:
-        error = run_test(filepath)
+        error = run_test(report, filepath)
         testname = test_get_name(filepath)
         if error:
             if error == "NO_CYCLES":
@@ -198,28 +326,33 @@ def run_all_tests(dirpath):
             failed_tests.append(testname)
         else:
             passed_tests.append(testname)
+        report.add_test(filepath, error)
     time_end = time.time()
     elapsed_ms = int((time_end - time_start) * 1000)
-    print("")
-    printMessage('SUCCESS', "==========",
-                 "{} tests from 1 test case ran. ({} ms total)" .
-                 format(len(all_files), elapsed_ms))
-    printMessage('SUCCESS', 'PASSED', "{} tests." .
-                 format(len(passed_tests)))
+    print_message("")
+    print_message("{} tests from 1 test case ran. ({} ms total)" .
+                  format(len(all_files), elapsed_ms),
+                  'SUCCESS', "==========")
+    print_message("{} tests." .
+                  format(len(passed_tests)),
+                  'SUCCESS', 'PASSED')
     if failed_tests:
-        printMessage('FAILURE', 'FAILED', "{} tests, listed below:" .
-                     format(len(failed_tests)))
+        print_message("{} tests, listed below:" .
+                     format(len(failed_tests)),
+                     'FAILURE', 'FAILED')
         failed_tests.sort()
         for test in failed_tests:
-            printMessage('FAILURE', "FAILED", "{}" . format(test))
-        return False
-    return True
+            print_message("{}" . format(test), 'FAILURE', "FAILED")
+
+    report.output()
+    return not bool(failed_tests)
 
 
 def create_argparse():
     parser = argparse.ArgumentParser()
     parser.add_argument("-blender", nargs="+")
     parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-outdir", nargs=1)
     parser.add_argument("-idiff", nargs=1)
     return parser
 
@@ -229,7 +362,7 @@ def main():
     args = parser.parse_args()
 
     global COLORS
-    global BLENDER, ROOT, IDIFF
+    global BLENDER, TESTDIR, IDIFF, OUTDIR
     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
     global VERBOSE
 
@@ -237,8 +370,12 @@ def main():
         COLORS = COLORS_ANSI
 
     BLENDER = args.blender[0]
-    ROOT = args.testdir[0]
+    TESTDIR = args.testdir[0]
     IDIFF = args.idiff[0]
+    OUTDIR = args.outdir[0]
+
+    if not os.path.exists(OUTDIR):
+        os.makedirs(OUTDIR)
 
     TEMP = tempfile.mkdtemp()
     TEMP_FILE_MASK = os.path.join(TEMP, "test")
@@ -248,7 +385,7 @@ def main():
 
     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
 
-    ok = run_all_tests(ROOT)
+    ok = run_all_tests(TESTDIR)
 
     # Cleanup temp files and folders
     if os.path.exists(TEMP_FILE):