c469c80264e5b8cba420d7e84fad304ff8bd298c
[blender.git] / release / scripts / io / netrender / client.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 import bpy
20 import sys, os, re
21 import http, http.client, http.server, urllib
22 import subprocess, shutil, time, hashlib
23 import json
24
25 import netrender
26 import netrender.model
27 import netrender.slave as slave
28 import netrender.master as master
29 from netrender.utils import *
30
31 def addFluidFiles(job, path):
32     if os.path.exists(path):
33         pattern = re.compile("fluidsurface_(final|preview)_([0-9]+)\.(bobj|bvel)\.gz")
34
35         for fluid_file in sorted(os.listdir(path)):
36             match = pattern.match(fluid_file)
37
38             if match:
39                 # fluid frames starts at 0, which explains the +1
40                 # This is stupid
41                 current_frame = int(match.groups()[1]) + 1
42                 job.addFile(path + fluid_file, current_frame, current_frame)
43
44 def addPointCache(job, ob, point_cache, default_path):
45     if not point_cache.use_disk_cache:
46         return
47
48
49     name = point_cache.name
50     if name == "":
51         name = "".join(["%02X" % ord(c) for c in ob.name])
52
53     cache_path = bpy.path.abspath(point_cache.filepath) if point_cache.use_external else default_path
54
55     index = "%02i" % point_cache.index
56
57     if os.path.exists(cache_path):
58         pattern = re.compile(name + "_([0-9]+)_" + index + "\.bphys")
59
60         cache_files = []
61
62         for cache_file in sorted(os.listdir(cache_path)):
63             match = pattern.match(cache_file)
64
65             if match:
66                 cache_frame = int(match.groups()[0])
67                 cache_files.append((cache_frame, cache_file))
68
69         cache_files.sort()
70
71         if len(cache_files) == 1:
72             cache_frame, cache_file = cache_files[0]
73             job.addFile(cache_path + cache_file, cache_frame, cache_frame)
74         else:
75             for i in range(len(cache_files)):
76                 current_item = cache_files[i]
77                 next_item = cache_files[i+1] if i + 1 < len(cache_files) else None
78                 previous_item = cache_files[i - 1] if i > 0 else None
79
80                 current_frame, current_file = current_item
81
82                 if  not next_item and not previous_item:
83                     job.addFile(cache_path + current_file, current_frame, current_frame)
84                 elif next_item and not previous_item:
85                     next_frame = next_item[0]
86                     job.addFile(cache_path + current_file, current_frame, next_frame - 1)
87                 elif not next_item and previous_item:
88                     previous_frame = previous_item[0]
89                     job.addFile(cache_path + current_file, previous_frame + 1, current_frame)
90                 else:
91                     next_frame = next_item[0]
92                     previous_frame = previous_item[0]
93                     job.addFile(cache_path + current_file, previous_frame + 1, next_frame - 1)
94
95 def fillCommonJobSettings(job, job_name, netsettings):
96     job.name = job_name
97     job.category = netsettings.job_category
98
99     for slave in netrender.blacklist:
100         job.blacklist.append(slave.id)
101
102     job.chunks = netsettings.chunks
103     job.priority = netsettings.priority
104     
105     if netsettings.job_type == "JOB_BLENDER":
106         job.type = netrender.model.JOB_BLENDER
107     elif netsettings.job_type == "JOB_PROCESS":
108         job.type = netrender.model.JOB_PROCESS
109     elif netsettings.job_type == "JOB_VCS":
110         job.type = netrender.model.JOB_VCS
111
112 def clientSendJob(conn, scene, anim = False):
113     netsettings = scene.network_render
114     if netsettings.job_type == "JOB_BLENDER":
115         return clientSendJobBlender(conn, scene, anim)
116     elif netsettings.job_type == "JOB_VCS":
117         return clientSendJobVCS(conn, scene, anim)
118
119 def clientSendJobVCS(conn, scene, anim = False):
120     netsettings = scene.network_render
121     job = netrender.model.RenderJob()
122
123     if anim:
124         for f in range(scene.frame_start, scene.frame_end + 1):
125             job.addFrame(f)
126     else:
127         job.addFrame(scene.frame_current)
128
129     filename = bpy.data.filepath
130     
131     if not filename.startswith(netsettings.vcs_wpath):
132         # this is an error, need better way to handle this
133         return
134
135     filename = filename[len(netsettings.vcs_wpath):]
136     
137     if filename[0] in (os.sep, os.altsep):
138         filename = filename[1:]
139     
140     print("CREATING VCS JOB", filename)
141     
142     job.addFile(filename, signed=False)
143
144     job_name = netsettings.job_name
145     path, name = os.path.split(filename)
146     if job_name == "[default]":
147         job_name = name
148
149
150     fillCommonJobSettings(job, job_name, netsettings)
151     
152     # VCS Specific code
153     job.version_info = netrender.model.VersioningInfo()
154     job.version_info.system = netsettings.vcs_system
155     job.version_info.wpath = netsettings.vcs_wpath
156     job.version_info.rpath = netsettings.vcs_rpath
157     job.version_info.revision = netsettings.vcs_revision
158
159     # try to send path first
160     conn.request("POST", "/job", json.dumps(job.serialize()))
161     response = conn.getresponse()
162     response.read()
163
164     job_id = response.getheader("job-id")
165     
166     # a VCS job is always good right now, need error handling
167
168     return job_id
169
170 def clientSendJobBlender(conn, scene, anim = False):
171     netsettings = scene.network_render
172     job = netrender.model.RenderJob()
173
174     if anim:
175         for f in range(scene.frame_start, scene.frame_end + 1):
176             job.addFrame(f)
177     else:
178         job.addFrame(scene.frame_current)
179
180     filename = bpy.data.filepath
181     job.addFile(filename)
182
183     job_name = netsettings.job_name
184     path, name = os.path.split(filename)
185     if job_name == "[default]":
186         job_name = name
187
188     ###########################
189     # LIBRARIES
190     ###########################
191     for lib in bpy.data.libraries:
192         file_path = bpy.path.abspath(lib.filepath)
193         if os.path.exists(file_path):
194             job.addFile(file_path)
195
196     ###########################
197     # IMAGES
198     ###########################
199     for image in bpy.data.images:
200         if image.source == "FILE" and not image.packed_file:
201             file_path = bpy.path.abspath(image.filepath)
202             if os.path.exists(file_path):
203                 job.addFile(file_path)
204                 
205                 tex_path = os.path.splitext(file_path)[0] + ".tex"
206                 if os.path.exists(tex_path):
207                     job.addFile(tex_path)
208
209     ###########################
210     # FLUID + POINT CACHE
211     ###########################
212     root, ext = os.path.splitext(name)
213     default_path = path + os.sep + "blendcache_" + root + os.sep # need an API call for that
214
215     for object in bpy.data.objects:
216         for modifier in object.modifiers:
217             if modifier.type == 'FLUID_SIMULATION' and modifier.settings.type == "DOMAIN":
218                 addFluidFiles(job, bpy.path.abspath(modifier.settings.path))
219             elif modifier.type == "CLOTH":
220                 addPointCache(job, object, modifier.point_cache, default_path)
221             elif modifier.type == "SOFT_BODY":
222                 addPointCache(job, object, modifier.point_cache, default_path)
223             elif modifier.type == "SMOKE" and modifier.smoke_type == "TYPE_DOMAIN":
224                 addPointCache(job, object, modifier.domain_settings.point_cache_low, default_path)
225                 if modifier.domain_settings.use_high_resolution:
226                     addPointCache(job, object, modifier.domain_settings.point_cache_high, default_path)
227             elif modifier.type == "MULTIRES" and modifier.is_external:
228                 file_path = bpy.path.abspath(modifier.filepath)
229                 job.addFile(file_path)
230
231         # particles modifier are stupid and don't contain data
232         # we have to go through the object property
233         for psys in object.particle_systems:
234             addPointCache(job, object, psys.point_cache, default_path)
235
236     #print(job.files)
237
238     fillCommonJobSettings(job, job_name, netsettings)
239
240     # try to send path first
241     conn.request("POST", "/job", json.dumps(job.serialize()))
242     response = conn.getresponse()
243     response.read()
244
245     job_id = response.getheader("job-id")
246
247     # if not ACCEPTED (but not processed), send files
248     if response.status == http.client.ACCEPTED:
249         for rfile in job.files:
250             f = open(rfile.filepath, "rb")
251             conn.request("PUT", fileURL(job_id, rfile.index), f)
252             f.close()
253             response = conn.getresponse()
254             response.read()
255
256     # server will reply with ACCEPTED until all files are found
257
258     return job_id
259
260 def requestResult(conn, job_id, frame):
261     conn.request("GET", renderURL(job_id, frame))
262
263 class NetworkRenderEngine(bpy.types.RenderEngine):
264     bl_idname = 'NET_RENDER'
265     bl_label = "Network Render"
266     bl_postprocess = False
267     def render(self, scene):
268         if scene.network_render.mode == "RENDER_CLIENT":
269             self.render_client(scene)
270         elif scene.network_render.mode == "RENDER_SLAVE":
271             self.render_slave(scene)
272         elif scene.network_render.mode == "RENDER_MASTER":
273             self.render_master(scene)
274         else:
275             print("UNKNOWN OPERATION MODE")
276
277     def render_master(self, scene):
278         netsettings = scene.network_render
279
280         address = "" if netsettings.server_address == "[default]" else netsettings.server_address
281
282         master.runMaster((address, netsettings.server_port), netsettings.use_master_broadcast, netsettings.use_master_clear, bpy.path.abspath(netsettings.path), self.update_stats, self.test_break)
283
284
285     def render_slave(self, scene):
286         slave.render_slave(self, scene.network_render, scene.render.threads)
287
288     def render_client(self, scene):
289         netsettings = scene.network_render
290         self.update_stats("", "Network render client initiation")
291
292
293         conn = clientConnection(netsettings.server_address, netsettings.server_port)
294
295         if conn:
296             # Sending file
297
298             self.update_stats("", "Network render exporting")
299
300             new_job = False
301
302             job_id = netsettings.job_id
303
304             # reading back result
305
306             self.update_stats("", "Network render waiting for results")
307             
308              
309             requestResult(conn, job_id, scene.frame_current)
310             response = conn.getresponse()
311             buf = response.read()
312
313             if response.status == http.client.NO_CONTENT:
314                 new_job = True
315                 netsettings.job_id = clientSendJob(conn, scene)
316                 job_id = netsettings.job_id
317
318                 requestResult(conn, job_id, scene.frame_current)
319                 response = conn.getresponse()
320                 buf = response.read()
321                 
322             while response.status == http.client.ACCEPTED and not self.test_break():
323                 time.sleep(1)
324                 requestResult(conn, job_id, scene.frame_current)
325                 response = conn.getresponse()
326                 buf = response.read()
327
328             # cancel new jobs (animate on network) on break
329             if self.test_break() and new_job:
330                 conn.request("POST", cancelURL(job_id))
331                 response = conn.getresponse()
332                 response.read()
333                 print( response.status, response.reason )
334                 netsettings.job_id = 0
335
336             if response.status != http.client.OK:
337                 conn.close()
338                 return
339
340             r = scene.render
341             x= int(r.resolution_x*r.resolution_percentage*0.01)
342             y= int(r.resolution_y*r.resolution_percentage*0.01)
343             
344             result_path = os.path.join(bpy.path.abspath(netsettings.path), "output.exr")
345             
346             folder = os.path.split(result_path)[0]
347             
348             if not os.path.exists(folder):
349                 os.mkdir(folder)
350
351             f = open(result_path, "wb")
352
353             f.write(buf)
354
355             f.close()
356
357             result = self.begin_result(0, 0, x, y)
358             result.load_from_file(result_path)
359             self.end_result(result)
360
361             conn.close()
362
363 def compatible(module):
364     module = __import__(module)
365     for subclass in module.__dict__.values():
366         try:            subclass.COMPAT_ENGINES.add('NET_RENDER')
367         except: pass
368     del module
369
370 #compatible("properties_render")
371 compatible("properties_world")
372 compatible("properties_material")
373 compatible("properties_data_mesh")
374 compatible("properties_data_camera")
375 compatible("properties_texture")