CTests: Add render tests for Cycles
authorSergey Sharybin <sergey.vfx@gmail.com>
Thu, 22 Jan 2015 10:53:49 +0000 (15:53 +0500)
committerSergey Sharybin <sergey.vfx@gmail.com>
Thu, 22 Jan 2015 10:57:20 +0000 (15:57 +0500)
The idea is to use the set of really small images from the lib folder
and run Cycles render on them comparing render output to reference
images in the tests repository.

For sure same thing could become more generic for BI or Freestyle
render engines.

Thanks Campbell for review and code tweaks!

tests/python/CMakeLists.txt
tests/python/cycles_render_tests.py [new file with mode: 0755]

index 85c68693792d9051d43c9db68502932535ad9925..59d3aa116197172aef87aaa792056738b24e5cc3 100644 (file)
@@ -48,6 +48,7 @@ else()
 endif()
 
 # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no 
+set(TEST_BLENDER_EXE_BARE ${TEST_BLENDER_EXE})
 set(TEST_BLENDER_EXE ${TEST_BLENDER_EXE} --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
 
 
@@ -358,3 +359,14 @@ add_test(export_fbx_all_objects ${TEST_BLENDER_EXE}
        --md5_source=${TEST_OUT_DIR}/export_fbx_all_objects.fbx
        --md5=b35eb2a9d0e73762ecae2278c25a38ac --md5_method=FILE
 )
+
+if(WITH_CYCLES)
+       if(OPENIMAGEIO_IDIFF)
+               add_test(cycles_shaders_test
+                       ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
+                       -blender "${TEST_BLENDER_EXE_BARE}"
+                       -testdir "${TEST_SRC_DIR}/cycles/ctests/shader"
+                       -idiff "${OPENIMAGEIO_IDIFF}"
+               )
+       endif()
+endif()
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
new file mode 100755 (executable)
index 0000000..815811f
--- /dev/null
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# Apache License, Version 2.0
+
+import argparse
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def render_file(filepath):
+    command = (
+        BLENDER,
+        "--background",
+        "-noaudio",
+        "--factory-startup",
+        filepath,
+        "-E", "CYCLES",
+        "-o", TEMP_FILE_MASK,
+        "-F", "PNG",
+        "-f", "1",
+        )
+    try:
+        output = subprocess.check_output(command)
+        if VERBOSE:
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        if os.path.exists(TEMP_FILE):
+            os.remove(TEMP_FILE)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        if b"Error: engine not found" in e.output:
+            return "NO_CYCLES"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
+    except:
+        if os.path.exists(TEMP_FILE):
+            os.remove(TEMP_FILE)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        return "CRASH"
+
+
+def test_get_name(filepath):
+    filename = os.path.basename(filepath)
+    return os.path.splitext(filename)[0]
+
+
+def verify_output(filepath):
+    testname = test_get_name(filepath)
+    dirpath = os.path.dirname(filepath)
+    reference_dirpath = os.path.join(dirpath, "reference_renders")
+    reference_image = os.path.join(reference_dirpath, testname + ".png")
+    if not os.path.exists(reference_image):
+        return False
+    command = (
+        IDIFF,
+        "-fail", "0.01",
+        "-failpercent", "1",
+        reference_image,
+        TEMP_FILE,
+        )
+    try:
+        subprocess.check_output(command)
+        return True
+    except subprocess.CalledProcessError as grepexc:
+        return grepexc.returncode == 1
+
+
+def run_test(filepath):
+    testname = test_get_name(filepath)
+    spacer = "." * (32 - len(testname))
+    print(testname, spacer, end="")
+    sys.stdout.flush()
+    error = render_file(filepath)
+    if not error:
+        if verify_output(filepath):
+            print("PASS")
+        else:
+            error = "VERIFY"
+    if error:
+        print("FAIL", error)
+    return error
+
+
+def blend_list(path):
+    for dirpath, dirnames, filenames in os.walk(path):
+        for filename in filenames:
+            if filename.lower().endswith(".blend"):
+                filepath = os.path.join(dirpath, filename)
+                yield filepath
+
+
+def run_all_tests(dirpath):
+    failed_tests = []
+    all_files = list(blend_list(dirpath))
+    all_files.sort()
+    for filepath in all_files:
+        error = run_test(filepath)
+        if error:
+            if error == "NO_CYCLES":
+                print("Can't perform tests because Cycles failed to load!")
+                return False
+            elif error == "NO_START":
+                print('Can not perform tests because blender fails to start.',
+                      'Make sure INSTALL target was run.')
+                return False
+            else:
+                print("Unknown error %r" % error)
+            testname = test_get_name(filepath)
+            failed_tests.append(testname)
+    if failed_tests:
+        failed_tests.sort()
+        print("\n\nFAILED tests:")
+        for test in failed_tests:
+            print("   ", test)
+        return False
+    return True
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    global BLENDER, ROOT, IDIFF
+    global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
+    global VERBOSE
+
+    BLENDER = args.blender[0]
+    ROOT = args.testdir[0]
+    IDIFF = args.idiff[0]
+
+    TEMP = tempfile.mkdtemp()
+    TEMP_FILE_MASK = os.path.join(TEMP, "test")
+    TEMP_FILE = TEMP_FILE_MASK + "0001.png"
+
+    TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
+
+    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
+
+    ok = run_all_tests(ROOT)
+
+    # Cleanup temp files and folders
+    if os.path.exists(TEMP_FILE):
+        os.remove(TEMP_FILE)
+    os.rmdir(TEMP)
+
+    sys.exit(not ok)
+
+
+if __name__ == "__main__":
+    main()