CTests: Initial work to cover Cycles nodes with OpenGL tests
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import os
6 import shutil
7 import subprocess
8 import sys
9 import time
10 import tempfile
11
12
13 class COLORS_ANSI:
14     RED = '\033[00;31m'
15     GREEN = '\033[00;32m'
16     ENDC = '\033[0m'
17
18
19 class COLORS_DUMMY:
20     RED = ''
21     GREEN = ''
22     ENDC = ''
23
24 COLORS = COLORS_DUMMY
25
26
27 def printMessage(type, status, message):
28     if type == 'SUCCESS':
29         print(COLORS.GREEN, end="")
30     elif type == 'FAILURE':
31         print(COLORS.RED, end="")
32     status_text = ...
33     if status == 'RUN':
34         status_text = " RUN      "
35     elif status == 'OK':
36         status_text = "       OK "
37     elif status == 'PASSED':
38         status_text = "  PASSED  "
39     elif status == 'FAILED':
40         status_text = "  FAILED  "
41     else:
42         status_text = status
43     print("[{}]" . format(status_text), end="")
44     print(COLORS.ENDC, end="")
45     print(" {}" . format(message))
46     sys.stdout.flush()
47
48
49 def render_file(filepath):
50     dirname = os.path.dirname(filepath)
51     basedir = os.path.dirname(dirname)
52     subject = os.path.basename(dirname)
53     if subject == 'opengl':
54         command = (
55             BLENDER,
56             "--window-geometry", "0", "0", "1", "1",
57             "-noaudio",
58             "--factory-startup",
59             "--enable-autoexec",
60             filepath,
61             "-E", "CYCLES",
62             # Run with OSL enabled
63             # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True",
64             "-o", TEMP_FILE_MASK,
65             "-F", "PNG",
66             '--python', os.path.join(basedir,
67                                      "util",
68                                      "render_opengl.py")
69         )
70     else:
71         command = (
72             BLENDER,
73             "--background",
74             "-noaudio",
75             "--factory-startup",
76             "--enable-autoexec",
77             filepath,
78             "-E", "CYCLES",
79             # Run with OSL enabled
80             # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True",
81             "-o", TEMP_FILE_MASK,
82             "-F", "PNG",
83             "-f", "1",
84             )
85     try:
86         output = subprocess.check_output(command)
87         if VERBOSE:
88             print(output.decode("utf-8"))
89         return None
90     except subprocess.CalledProcessError as e:
91         if os.path.exists(TEMP_FILE):
92             os.remove(TEMP_FILE)
93         if VERBOSE:
94             print(e.output.decode("utf-8"))
95         if b"Error: engine not found" in e.output:
96             return "NO_CYCLES"
97         elif b"blender probably wont start" in e.output:
98             return "NO_START"
99         return "CRASH"
100     except BaseException as e:
101         if os.path.exists(TEMP_FILE):
102             os.remove(TEMP_FILE)
103         if VERBOSE:
104             print(e)
105         return "CRASH"
106
107
108 def test_get_name(filepath):
109     filename = os.path.basename(filepath)
110     return os.path.splitext(filename)[0]
111
112
113 def verify_output(filepath):
114     testname = test_get_name(filepath)
115     dirpath = os.path.dirname(filepath)
116     reference_dirpath = os.path.join(dirpath, "reference_renders")
117     reference_image = os.path.join(reference_dirpath, testname + ".png")
118     failed_image = os.path.join(reference_dirpath, testname + ".fail.png")
119     if not os.path.exists(reference_image):
120         return False
121     command = (
122         IDIFF,
123         "-fail", "0.015",
124         "-failpercent", "1",
125         reference_image,
126         TEMP_FILE,
127         )
128     try:
129         subprocess.check_output(command)
130         failed = False
131     except subprocess.CalledProcessError as e:
132         if VERBOSE:
133             print(e.output.decode("utf-8"))
134         failed = e.returncode != 1
135     if failed:
136         shutil.copy(TEMP_FILE, failed_image)
137     elif os.path.exists(failed_image):
138         os.remove(failed_image)
139     return not failed
140
141
142 def run_test(filepath):
143     testname = test_get_name(filepath)
144     spacer = "." * (32 - len(testname))
145     printMessage('SUCCESS', 'RUN', testname)
146     time_start = time.time()
147     error = render_file(filepath)
148     status = "FAIL"
149     if not error:
150         if not verify_output(filepath):
151             error = "VERIFY"
152     time_end = time.time()
153     elapsed_ms = int((time_end - time_start) * 1000)
154     if not error:
155         printMessage('SUCCESS', 'OK', "{} ({} ms)" .
156                      format(testname, elapsed_ms))
157     else:
158         if error == "NO_CYCLES":
159             print("Can't perform tests because Cycles failed to load!")
160             return False
161         elif error == "NO_START":
162             print('Can not perform tests because blender fails to start.',
163                   'Make sure INSTALL target was run.')
164             return False
165         elif error == 'VERIFY':
166             print("Rendered result is different from reference image")
167         else:
168             print("Unknown error %r" % error)
169         printMessage('FAILURE', 'FAILED', "{} ({} ms)" .
170                      format(testname, elapsed_ms))
171     return error
172
173
174 def blend_list(path):
175     for dirpath, dirnames, filenames in os.walk(path):
176         for filename in filenames:
177             if filename.lower().endswith(".blend"):
178                 filepath = os.path.join(dirpath, filename)
179                 yield filepath
180
181
182 def run_all_tests(dirpath):
183     passed_tests = []
184     failed_tests = []
185     all_files = list(blend_list(dirpath))
186     all_files.sort()
187     printMessage('SUCCESS', "==========",
188                  "Running {} tests from 1 test case." . format(len(all_files)))
189     time_start = time.time()
190     for filepath in all_files:
191         error = run_test(filepath)
192         testname = test_get_name(filepath)
193         if error:
194             if error == "NO_CYCLES":
195                 return False
196             elif error == "NO_START":
197                 return False
198             failed_tests.append(testname)
199         else:
200             passed_tests.append(testname)
201     time_end = time.time()
202     elapsed_ms = int((time_end - time_start) * 1000)
203     print("")
204     printMessage('SUCCESS', "==========",
205                  "{} tests from 1 test case ran. ({} ms total)" .
206                  format(len(all_files), elapsed_ms))
207     printMessage('SUCCESS', 'PASSED', "{} tests." .
208                  format(len(passed_tests)))
209     if failed_tests:
210         printMessage('FAILURE', 'FAILED', "{} tests, listed below:" .
211                      format(len(failed_tests)))
212         failed_tests.sort()
213         for test in failed_tests:
214             printMessage('FAILURE', "FAILED", "{}" . format(test))
215         return False
216     return True
217
218
219 def create_argparse():
220     parser = argparse.ArgumentParser()
221     parser.add_argument("-blender", nargs="+")
222     parser.add_argument("-testdir", nargs=1)
223     parser.add_argument("-idiff", nargs=1)
224     return parser
225
226
227 def main():
228     parser = create_argparse()
229     args = parser.parse_args()
230
231     global COLORS
232     global BLENDER, ROOT, IDIFF
233     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
234     global VERBOSE
235
236     if os.environ.get("CYCLESTEST_COLOR") is not None:
237         COLORS = COLORS_ANSI
238
239     BLENDER = args.blender[0]
240     ROOT = args.testdir[0]
241     IDIFF = args.idiff[0]
242
243     TEMP = tempfile.mkdtemp()
244     TEMP_FILE_MASK = os.path.join(TEMP, "test")
245     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
246
247     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
248
249     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
250
251     ok = run_all_tests(ROOT)
252
253     # Cleanup temp files and folders
254     if os.path.exists(TEMP_FILE):
255         os.remove(TEMP_FILE)
256     os.rmdir(TEMP)
257
258     sys.exit(not ok)
259
260
261 if __name__ == "__main__":
262     main()