64a71da301ae064002db6ca2c25e94027cf221b7
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import os
6 import shutil
7 import subprocess
8 import sys
9 import time
10 import tempfile
11
12
13 class COLORS_ANSI:
14     RED = '\033[00;31m'
15     GREEN = '\033[00;32m'
16     ENDC = '\033[0m'
17
18
19 class COLORS_DUMMY:
20     RED = ''
21     GREEN = ''
22     ENDC = ''
23
24 COLORS = COLORS_DUMMY
25
26
27 def printMessage(type, status, message):
28     if type == 'SUCCESS':
29         print(COLORS.GREEN, end="")
30     elif type == 'FAILURE':
31         print(COLORS.RED, end="")
32     status_text = ...
33     if status == 'RUN':
34         status_text = " RUN      "
35     elif status == 'OK':
36         status_text = "       OK "
37     elif status == 'PASSED':
38         status_text = "  PASSED  "
39     elif status == 'FAILED':
40         status_text = "  FAILED  "
41     else:
42         status_text = status
43     print("[{}]" . format(status_text), end="")
44     print(COLORS.ENDC, end="")
45     print(" {}" . format(message))
46     sys.stdout.flush()
47
48
49 def render_file(filepath):
50     command = (
51         BLENDER,
52         "--background",
53         "-noaudio",
54         "--factory-startup",
55         "--enable-autoexec",
56         filepath,
57         "-E", "CYCLES",
58         # Run with OSL enabled
59         # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True",
60         "-o", TEMP_FILE_MASK,
61         "-F", "PNG",
62         "-f", "1",
63         )
64     try:
65         output = subprocess.check_output(command)
66         if VERBOSE:
67             print(output.decode("utf-8"))
68         return None
69     except subprocess.CalledProcessError as e:
70         if os.path.exists(TEMP_FILE):
71             os.remove(TEMP_FILE)
72         if VERBOSE:
73             print(e.output.decode("utf-8"))
74         if b"Error: engine not found" in e.output:
75             return "NO_CYCLES"
76         elif b"blender probably wont start" in e.output:
77             return "NO_START"
78         return "CRASH"
79     except BaseException as e:
80         if os.path.exists(TEMP_FILE):
81             os.remove(TEMP_FILE)
82         if VERBOSE:
83             print(e)
84         return "CRASH"
85
86
87 def test_get_name(filepath):
88     filename = os.path.basename(filepath)
89     return os.path.splitext(filename)[0]
90
91
92 def verify_output(filepath):
93     testname = test_get_name(filepath)
94     dirpath = os.path.dirname(filepath)
95     reference_dirpath = os.path.join(dirpath, "reference_renders")
96     reference_image = os.path.join(reference_dirpath, testname + ".png")
97     failed_image = os.path.join(reference_dirpath, testname + ".fail.png")
98     if not os.path.exists(reference_image):
99         return False
100     command = (
101         IDIFF,
102         "-fail", "0.015",
103         "-failpercent", "1",
104         reference_image,
105         TEMP_FILE,
106         )
107     try:
108         subprocess.check_output(command)
109         failed = False
110     except subprocess.CalledProcessError as e:
111         if VERBOSE:
112             print(e.output.decode("utf-8"))
113         failed = e.returncode != 1
114     if failed:
115         shutil.copy(TEMP_FILE, failed_image)
116     elif os.path.exists(failed_image):
117         os.remove(failed_image)
118     return not failed
119
120
121 def run_test(filepath):
122     testname = test_get_name(filepath)
123     spacer = "." * (32 - len(testname))
124     printMessage('SUCCESS', 'RUN', testname)
125     time_start = time.time()
126     error = render_file(filepath)
127     status = "FAIL"
128     if not error:
129         if not verify_output(filepath):
130             error = "VERIFY"
131     time_end = time.time()
132     elapsed_ms = int((time_end - time_start) * 1000)
133     if not error:
134         printMessage('SUCCESS', 'OK', "{} ({} ms)" .
135                      format(testname, elapsed_ms))
136     else:
137         if error == "NO_CYCLES":
138             print("Can't perform tests because Cycles failed to load!")
139             return False
140         elif error == "NO_START":
141             print('Can not perform tests because blender fails to start.',
142                   'Make sure INSTALL target was run.')
143             return False
144         elif error == 'VERIFY':
145             print("Rendered result is different from reference image")
146         else:
147             print("Unknown error %r" % error)
148         printMessage('FAILURE', 'FAILED', "{} ({} ms)" .
149                      format(testname, elapsed_ms))
150     return error
151
152
153 def blend_list(path):
154     for dirpath, dirnames, filenames in os.walk(path):
155         for filename in filenames:
156             if filename.lower().endswith(".blend"):
157                 filepath = os.path.join(dirpath, filename)
158                 yield filepath
159
160
161 def run_all_tests(dirpath):
162     passed_tests = []
163     failed_tests = []
164     all_files = list(blend_list(dirpath))
165     all_files.sort()
166     printMessage('SUCCESS', "==========",
167                  "Running {} tests from 1 test case." . format(len(all_files)))
168     time_start = time.time()
169     for filepath in all_files:
170         error = run_test(filepath)
171         testname = test_get_name(filepath)
172         if error:
173             if error == "NO_CYCLES":
174                 return False
175             elif error == "NO_START":
176                 return False
177             failed_tests.append(testname)
178         else:
179             passed_tests.append(testname)
180     time_end = time.time()
181     elapsed_ms = int((time_end - time_start) * 1000)
182     print("")
183     printMessage('SUCCESS', "==========",
184                  "{} tests from 1 test case ran. ({} ms total)" .
185                  format(len(all_files), elapsed_ms))
186     printMessage('SUCCESS', 'PASSED', "{} tests." .
187                  format(len(passed_tests)))
188     if failed_tests:
189         printMessage('FAILURE', 'FAILED', "{} tests, listed below:" .
190                      format(len(failed_tests)))
191         failed_tests.sort()
192         for test in failed_tests:
193             printMessage('FAILURE', "FAILED", "{}" . format(test))
194         return False
195     return True
196
197
198 def create_argparse():
199     parser = argparse.ArgumentParser()
200     parser.add_argument("-blender", nargs="+")
201     parser.add_argument("-testdir", nargs=1)
202     parser.add_argument("-idiff", nargs=1)
203     return parser
204
205
206 def main():
207     parser = create_argparse()
208     args = parser.parse_args()
209
210     global COLORS
211     global BLENDER, ROOT, IDIFF
212     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
213     global VERBOSE
214
215     if os.environ.get("CYCLESTEST_COLOR") is not None:
216         COLORS = COLORS_ANSI
217
218     BLENDER = args.blender[0]
219     ROOT = args.testdir[0]
220     IDIFF = args.idiff[0]
221
222     TEMP = tempfile.mkdtemp()
223     TEMP_FILE_MASK = os.path.join(TEMP, "test")
224     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
225
226     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
227
228     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
229
230     ok = run_all_tests(ROOT)
231
232     # Cleanup temp files and folders
233     if os.path.exists(TEMP_FILE):
234         os.remove(TEMP_FILE)
235     os.rmdir(TEMP)
236
237     sys.exit(not ok)
238
239
240 if __name__ == "__main__":
241     main()