Add extra Cycles regression tests
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import os
6 import subprocess
7 import sys
8 import tempfile
9
10
11 def render_file(filepath):
12     command = (
13         BLENDER,
14         "--background",
15         "-noaudio",
16         "--factory-startup",
17         filepath,
18         "-E", "CYCLES",
19         "-o", TEMP_FILE_MASK,
20         "-F", "PNG",
21         "-f", "1",
22         )
23     try:
24         output = subprocess.check_output(command)
25         if VERBOSE:
26             print(output.decode("utf-8"))
27         return None
28     except subprocess.CalledProcessError as e:
29         if os.path.exists(TEMP_FILE):
30             os.remove(TEMP_FILE)
31         if VERBOSE:
32             print(e.output.decode("utf-8"))
33         if b"Error: engine not found" in e.output:
34             return "NO_CYCLES"
35         elif b"blender probably wont start" in e.output:
36             return "NO_START"
37         return "CRASH"
38     except:
39         if os.path.exists(TEMP_FILE):
40             os.remove(TEMP_FILE)
41         if VERBOSE:
42             print(e.output.decode("utf-8"))
43         return "CRASH"
44
45
46 def test_get_name(filepath):
47     filename = os.path.basename(filepath)
48     return os.path.splitext(filename)[0]
49
50
51 def verify_output(filepath):
52     testname = test_get_name(filepath)
53     dirpath = os.path.dirname(filepath)
54     reference_dirpath = os.path.join(dirpath, "reference_renders")
55     reference_image = os.path.join(reference_dirpath, testname + ".png")
56     if not os.path.exists(reference_image):
57         return False
58     command = (
59         IDIFF,
60         "-fail", "0.01",
61         "-failpercent", "1",
62         reference_image,
63         TEMP_FILE,
64         )
65     try:
66         subprocess.check_output(command)
67         return True
68     except subprocess.CalledProcessError as e:
69         if VERBOSE:
70             print(e.output.decode("utf-8"))
71         return e.returncode == 1
72
73
74 def run_test(filepath):
75     testname = test_get_name(filepath)
76     spacer = "." * (32 - len(testname))
77     print(testname, spacer, end="")
78     sys.stdout.flush()
79     error = render_file(filepath)
80     if not error:
81         if verify_output(filepath):
82             print("PASS")
83         else:
84             error = "VERIFY"
85     if error:
86         print("FAIL", error)
87     return error
88
89
90 def blend_list(path):
91     for dirpath, dirnames, filenames in os.walk(path):
92         for filename in filenames:
93             if filename.lower().endswith(".blend"):
94                 filepath = os.path.join(dirpath, filename)
95                 yield filepath
96
97
98 def run_all_tests(dirpath):
99     failed_tests = []
100     all_files = list(blend_list(dirpath))
101     all_files.sort()
102     for filepath in all_files:
103         error = run_test(filepath)
104         if error:
105             if error == "NO_CYCLES":
106                 print("Can't perform tests because Cycles failed to load!")
107                 return False
108             elif error == "NO_START":
109                 print('Can not perform tests because blender fails to start.',
110                       'Make sure INSTALL target was run.')
111                 return False
112             elif error == 'VERIFY':
113                 pass
114             else:
115                 print("Unknown error %r" % error)
116             testname = test_get_name(filepath)
117             failed_tests.append(testname)
118     if failed_tests:
119         failed_tests.sort()
120         print("\n\nFAILED tests:")
121         for test in failed_tests:
122             print("   ", test)
123         return False
124     return True
125
126
127 def create_argparse():
128     parser = argparse.ArgumentParser()
129     parser.add_argument("-blender", nargs="+")
130     parser.add_argument("-testdir", nargs=1)
131     parser.add_argument("-idiff", nargs=1)
132     return parser
133
134
135 def main():
136     parser = create_argparse()
137     args = parser.parse_args()
138
139     global BLENDER, ROOT, IDIFF
140     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
141     global VERBOSE
142
143     BLENDER = args.blender[0]
144     ROOT = args.testdir[0]
145     IDIFF = args.idiff[0]
146
147     TEMP = tempfile.mkdtemp()
148     TEMP_FILE_MASK = os.path.join(TEMP, "test")
149     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
150
151     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
152
153     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
154
155     ok = run_all_tests(ROOT)
156
157     # Cleanup temp files and folders
158     if os.path.exists(TEMP_FILE):
159         os.remove(TEMP_FILE)
160     os.rmdir(TEMP)
161
162     sys.exit(not ok)
163
164
165 if __name__ == "__main__":
166     main()