Revert "New cycles tests require buildbot to checkout tests of have test succeed...
[blender.git] / tests / python / cycles_render_tests.py
1 #!/usr/bin/env python3
2 # Apache License, Version 2.0
3
4 import argparse
5 import os
6 import subprocess
7 import sys
8 import tempfile
9
10
11 def render_file(filepath):
12     command = (
13         BLENDER,
14         "--background",
15         "-noaudio",
16         "--factory-startup",
17         filepath,
18         "-E", "CYCLES",
19         "-o", TEMP_FILE_MASK,
20         "-F", "PNG",
21         "-f", "1",
22         )
23     try:
24         output = subprocess.check_output(command)
25         if VERBOSE:
26             print(output.decode("utf-8"))
27         return None
28     except subprocess.CalledProcessError as e:
29         if os.path.exists(TEMP_FILE):
30             os.remove(TEMP_FILE)
31         if VERBOSE:
32             print(e.output.decode("utf-8"))
33         if b"Error: engine not found" in e.output:
34             return "NO_CYCLES"
35         elif b"blender probably wont start" in e.output:
36             return "NO_START"
37         return "CRASH"
38     except:
39         if os.path.exists(TEMP_FILE):
40             os.remove(TEMP_FILE)
41         if VERBOSE:
42             print(e.output.decode("utf-8"))
43         return "CRASH"
44
45
46 def test_get_name(filepath):
47     filename = os.path.basename(filepath)
48     return os.path.splitext(filename)[0]
49
50
51 def verify_output(filepath):
52     testname = test_get_name(filepath)
53     dirpath = os.path.dirname(filepath)
54     reference_dirpath = os.path.join(dirpath, "reference_renders")
55     reference_image = os.path.join(reference_dirpath, testname + ".png")
56     if not os.path.exists(reference_image):
57         return False
58     command = (
59         IDIFF,
60         "-fail", "0.01",
61         "-failpercent", "1",
62         reference_image,
63         TEMP_FILE,
64         )
65     try:
66         subprocess.check_output(command)
67         return True
68     except subprocess.CalledProcessError as grepexc:
69         return grepexc.returncode == 1
70
71
72 def run_test(filepath):
73     testname = test_get_name(filepath)
74     spacer = "." * (32 - len(testname))
75     print(testname, spacer, end="")
76     sys.stdout.flush()
77     error = render_file(filepath)
78     if not error:
79         if verify_output(filepath):
80             print("PASS")
81         else:
82             error = "VERIFY"
83     if error:
84         print("FAIL", error)
85     return error
86
87
88 def blend_list(path):
89     for dirpath, dirnames, filenames in os.walk(path):
90         for filename in filenames:
91             if filename.lower().endswith(".blend"):
92                 filepath = os.path.join(dirpath, filename)
93                 yield filepath
94
95
96 def run_all_tests(dirpath):
97     failed_tests = []
98     all_files = list(blend_list(dirpath))
99     all_files.sort()
100     for filepath in all_files:
101         error = run_test(filepath)
102         if error:
103             if error == "NO_CYCLES":
104                 print("Can't perform tests because Cycles failed to load!")
105                 return False
106             elif error == "NO_START":
107                 print('Can not perform tests because blender fails to start.',
108                       'Make sure INSTALL target was run.')
109                 return False
110             else:
111                 print("Unknown error %r" % error)
112             testname = test_get_name(filepath)
113             failed_tests.append(testname)
114     if failed_tests:
115         failed_tests.sort()
116         print("\n\nFAILED tests:")
117         for test in failed_tests:
118             print("   ", test)
119         return False
120     return True
121
122
123 def create_argparse():
124     parser = argparse.ArgumentParser()
125     parser.add_argument("-blender", nargs="+")
126     parser.add_argument("-testdir", nargs=1)
127     parser.add_argument("-idiff", nargs=1)
128     return parser
129
130
131 def main():
132     parser = create_argparse()
133     args = parser.parse_args()
134
135     global BLENDER, ROOT, IDIFF
136     global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
137     global VERBOSE
138
139     BLENDER = args.blender[0]
140     ROOT = args.testdir[0]
141     IDIFF = args.idiff[0]
142
143     TEMP = tempfile.mkdtemp()
144     TEMP_FILE_MASK = os.path.join(TEMP, "test")
145     TEMP_FILE = TEMP_FILE_MASK + "0001.png"
146
147     TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
148
149     VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
150
151     ok = run_all_tests(ROOT)
152
153     # Cleanup temp files and folders
154     if os.path.exists(TEMP_FILE):
155         os.remove(TEMP_FILE)
156     os.rmdir(TEMP)
157
158     sys.exit(not ok)
159
160
161 if __name__ == "__main__":
162     main()