Tests: add OpenGL UI drawing tests.
authorBrecht Van Lommel <brechtvanlommel@gmail.com>
Wed, 14 Feb 2018 19:33:33 +0000 (20:33 +0100)
committerBrecht Van Lommel <brechtvanlommel@gmail.com>
Fri, 16 Feb 2018 11:51:49 +0000 (12:51 +0100)
This reuses the Cycles regression test code to also work for OpenGL UI drawing.
We launch Blender with a bunch of .blend files, take a screenshot and compare
it with a reference screenshot, and generate a HMTL report showing the failed
tests and their differences.

For Cycles we keep small reference renders to compare to in svn, but for OpenGL
developers currently have to generate the references manually. How to use:

* WITH_OPENGL_DRAW_TESTS=ON in CMake
* BLENDER_TEST_UPDATE=1 ctest -R opengl_draw
* .. make code changes ..
* ctest -R opengl_draw
* open build_dir/tests/opengl_draw/report.html

Differential Revision: https://developer.blender.org/D3064

CMakeLists.txt
tests/python/CMakeLists.txt
tests/python/modules/render_report.py
tests/python/opengl_draw_tests.py [new file with mode: 0755]

index 5b2bd2fd460a294bd50acc95f02218301ac64f1e..dce622d4778aa0769db073b2e39538b343dc4448 100644 (file)
@@ -458,7 +458,8 @@ option(WITH_BOOST                                   "Enable features depending on boost" ON)
 
 # Unit testsing
 option(WITH_GTESTS "Enable GTest unit testing" OFF)
-option(WITH_OPENGL_TESTS "Enable OpenGL related unit testing (Experimental)" OFF)
+option(WITH_OPENGL_RENDER_TESTS "Enable OpenGL render related unit testing (Experimental)" OFF)
+option(WITH_OPENGL_DRAW_TESTS "Enable OpenGL UI drawing related unit testing (Experimental)" OFF)
 
 
 # Documentation
index 662a35558b7e09fb529ef3e1b958b7b6c73c726c..ceed2ca4d09fc2f9f752648537139838e2dc290b 100644 (file)
@@ -538,7 +538,7 @@ if(WITH_CYCLES)
                                )
                        endif()
                endmacro()
-               if(WITH_OPENGL_TESTS)
+               if(WITH_OPENGL_RENDER_TESTS)
                        add_cycles_render_test(opengl)
                endif()
                add_cycles_render_test(bake)
@@ -562,6 +562,52 @@ if(WITH_CYCLES)
        endif()
 endif()
 
+if(WITH_OPENGL_DRAW_TESTS)
+       if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/opengl")
+               macro(add_opengl_draw_test subject)
+                       if(MSVC)
+                               add_test(
+                                       NAME opengl_draw_${subject}_test
+                                       COMMAND
+                                               "$<TARGET_FILE_DIR:blender>/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$<CONFIG:Debug>:_d>"
+                                               ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py
+                                       -blender "$<TARGET_FILE:blender>"
+                                       -testdir "${TEST_SRC_DIR}/${subject}"
+                                       -idiff "${OPENIMAGEIO_IDIFF}"
+                                       -outdir "${TEST_OUT_DIR}/opengl_draw"
+                               )
+                       else()
+                               add_test(
+                                       NAME opengl_draw_${subject}_test
+                                       COMMAND ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py
+                                       -blender "$<TARGET_FILE:blender>"
+                                       -testdir "${TEST_SRC_DIR}/${subject}"
+                                       -idiff "${OPENIMAGEIO_IDIFF}"
+                                       -outdir "${TEST_OUT_DIR}/opengl_draw"
+                               )
+                       endif()
+               endmacro()
+
+               function(add_opengl_draw_tests)
+                       # Use all test folders
+                       file(GLOB children RELATIVE ${TEST_SRC_DIR} ${TEST_SRC_DIR}/*)
+                       foreach(child ${children})
+                               if(IS_DIRECTORY ${TEST_SRC_DIR}/${child})
+                                       file(GLOB blends ${TEST_SRC_DIR}/${child}/*.blend)
+                                       if(blends)
+                                               add_opengl_draw_test(${child})
+                                       endif()
+                               endif()
+                       endforeach()
+               endfunction()
+
+               add_opengl_draw_tests()
+       else()
+               MESSAGE(STATUS "Disabling OpenGL tests because tests folder does not exist")
+       endif()
+endif()
+
+
 if(WITH_ALEMBIC)
        find_package_wrapper(Alembic)
        if(NOT ALEMBIC_FOUND)
index 930a08282e870ad03400b1079292427fbab2f185..5ccd5076fbc387195da292902fc0fedf0a585d7a 100755 (executable)
@@ -49,12 +49,11 @@ def print_message(message, type=None, status=''):
     sys.stdout.flush()
 
 
-def blend_list(path):
-    for dirpath, dirnames, filenames in os.walk(path):
-        for filename in filenames:
-            if filename.lower().endswith(".blend"):
-                filepath = os.path.join(dirpath, filename)
-                yield filepath
+def blend_list(dirpath):
+    for filename in os.listdir(dirpath):
+        if filename.lower().endswith(".blend"):
+            filepath = os.path.join(dirpath, filename)
+            yield filepath
 
 def test_get_name(filepath):
     filename = os.path.basename(filepath)
diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py
new file mode 100755 (executable)
index 0000000..9993045
--- /dev/null
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# Apache License, Version 2.0
+
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+
+def screenshot():
+    import bpy
+
+    output_path = sys.argv[-1]
+
+    # Force redraw and take screenshot.
+    bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
+    bpy.ops.screen.screenshot(filepath=output_path, full=True)
+
+    bpy.ops.wm.quit_blender()
+
+# When run from inside Blender, take screenshot and exit.
+try:
+    import bpy
+    inside_blender = True
+except ImportError:
+    inside_blender = False
+
+if inside_blender:
+    screenshot()
+    sys.exit(0)
+
+
+def render_file(filepath, output_filepath):
+    command = (
+        BLENDER,
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        filepath,
+        "-P",
+        os.path.realpath(__file__),
+        "--",
+        output_filepath)
+
+    try:
+        # Success
+        output = subprocess.check_output(command)
+        if VERBOSE:
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        # Error
+        if os.path.exists(output_filepath):
+            os.remove(output_filepath)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        return "CRASH"
+    except BaseException as e:
+        # Crash
+        if os.path.exists(output_filepath):
+            os.remove(output_filepath)
+        if VERBOSE:
+            print(e)
+        return "CRASH"
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-outdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    global BLENDER, VERBOSE
+
+    BLENDER = args.blender[0]
+    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
+
+    test_dir = args.testdir[0]
+    idiff = args.idiff[0]
+    output_dir = args.outdir[0]
+
+    from modules import render_report
+    report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff)
+    ok = report.run(test_dir, render_file)
+
+    sys.exit(not ok)
+
+if __name__ == "__main__":
+    main()