netrender
[blender.git] / release / scripts / io / netrender / client.py
1 # ##### BEGIN GPL LICENSE BLOCK #####
2 #
3 #  This program is free software; you can redistribute it and/or
4 #  modify it under the terms of the GNU General Public License
5 #  as published by the Free Software Foundation; either version 2
6 #  of the License, or (at your option) any later version.
7 #
8 #  This program is distributed in the hope that it will be useful,
9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 #  GNU General Public License for more details.
12 #
13 #  You should have received a copy of the GNU General Public License
14 #  along with this program; if not, write to the Free Software Foundation,
15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 #
17 # ##### END GPL LICENSE BLOCK #####
18
19 import bpy
20 import sys, os, re
21 import http, http.client, http.server, urllib
22 import subprocess, shutil, time, hashlib
23 import json
24
25 import netrender
26 import netrender.model
27 import netrender.slave as slave
28 import netrender.master as master
29 from netrender.utils import *
30
31 def addFluidFiles(job, path):
32     if os.path.exists(path):
33         pattern = re.compile("fluidsurface_(final|preview)_([0-9]+)\.(bobj|bvel)\.gz")
34
35         for fluid_file in sorted(os.listdir(path)):
36             match = pattern.match(fluid_file)
37
38             if match:
39                 # fluid frames starts at 0, which explains the +1
40                 # This is stupid
41                 current_frame = int(match.groups()[1]) + 1
42                 job.addFile(path + fluid_file, current_frame, current_frame)
43
44 def addPointCache(job, ob, point_cache, default_path):
45     if not point_cache.use_disk_cache:
46         return
47
48
49     name = point_cache.name
50     if name == "":
51         name = "".join(["%02X" % ord(c) for c in ob.name])
52
53     cache_path = bpy.path.abspath(point_cache.filepath) if point_cache.use_external else default_path
54
55     index = "%02i" % point_cache.index
56
57     if os.path.exists(cache_path):
58         pattern = re.compile(name + "_([0-9]+)_" + index + "\.bphys")
59
60         cache_files = []
61
62         for cache_file in sorted(os.listdir(cache_path)):
63             match = pattern.match(cache_file)
64
65             if match:
66                 cache_frame = int(match.groups()[0])
67                 cache_files.append((cache_frame, cache_file))
68
69         cache_files.sort()
70
71         if len(cache_files) == 1:
72             cache_frame, cache_file = cache_files[0]
73             job.addFile(cache_path + cache_file, cache_frame, cache_frame)
74         else:
75             for i in range(len(cache_files)):
76                 current_item = cache_files[i]
77                 next_item = cache_files[i+1] if i + 1 < len(cache_files) else None
78                 previous_item = cache_files[i - 1] if i > 0 else None
79
80                 current_frame, current_file = current_item
81
82                 if  not next_item and not previous_item:
83                     job.addFile(cache_path + current_file, current_frame, current_frame)
84                 elif next_item and not previous_item:
85                     next_frame = next_item[0]
86                     job.addFile(cache_path + current_file, current_frame, next_frame - 1)
87                 elif not next_item and previous_item:
88                     previous_frame = previous_item[0]
89                     job.addFile(cache_path + current_file, previous_frame + 1, current_frame)
90                 else:
91                     next_frame = next_item[0]
92                     previous_frame = previous_item[0]
93                     job.addFile(cache_path + current_file, previous_frame + 1, next_frame - 1)
94
95 def fillCommonJobSettings(job, job_name, netsettings):
96     job.name = job_name
97     job.category = netsettings.job_category
98
99     for slave in netrender.blacklist:
100         job.blacklist.append(slave.id)
101
102     job.chunks = netsettings.chunks
103     job.priority = netsettings.priority
104     
105     if netsettings.job_type == "JOB_BLENDER":
106         job.type = netrender.model.JOB_BLENDER
107     elif netsettings.job_type == "JOB_PROCESS":
108         job.type = netrender.model.JOB_PROCESS
109     elif netsettings.job_type == "JOB_VCS":
110         job.type = netrender.model.JOB_VCS
111
112 def clientSendJob(conn, scene, anim = False):
113     netsettings = scene.network_render
114     if netsettings.job_type == "JOB_BLENDER":
115         return clientSendJobBlender(conn, scene, anim)
116     elif netsettings.job_type == "JOB_VCS":
117         return clientSendJobVCS(conn, scene, anim)
118
119 def clientSendJobVCS(conn, scene, anim = False):
120     netsettings = scene.network_render
121     job = netrender.model.RenderJob()
122
123     if anim:
124         for f in range(scene.frame_start, scene.frame_end + 1):
125             job.addFrame(f)
126     else:
127         job.addFrame(scene.frame_current)
128
129     filename = bpy.data.filepath
130     
131     if not filename.startswith(netsettings.vcs_wpath):
132         # this is an error, need better way to handle this
133         return
134
135     filename = filename[len(netsettings.vcs_wpath):]
136     
137     if filename[0] in (os.sep, os.altsep):
138         filename = filename[1:]
139     
140     print("CREATING VCS JOB", filename)
141     
142     job.addFile(filename, signed=False)
143
144     job_name = netsettings.job_name
145     path, name = os.path.split(filename)
146     if job_name == "[default]":
147         job_name = name
148
149
150     fillCommonJobSettings(job, job_name, netsettings)
151     
152     # VCS Specific code
153     job.version_info = netrender.model.VersioningInfo()
154     job.version_info.system = netsettings.vcs_system
155     job.version_info.wpath = netsettings.vcs_wpath
156     job.version_info.rpath = netsettings.vcs_rpath
157     job.version_info.revision = netsettings.vcs_revision
158
159     # try to send path first
160     conn.request("POST", "/job", json.dumps(job.serialize()))
161     response = conn.getresponse()
162     response.read()
163
164     job_id = response.getheader("job-id")
165     
166     # a VCS job is always good right now, need error handling
167
168     return job_id
169
170 def clientSendJobBlender(conn, scene, anim = False):
171     netsettings = scene.network_render
172     job = netrender.model.RenderJob()
173
174     if anim:
175         for f in range(scene.frame_start, scene.frame_end + 1):
176             job.addFrame(f)
177     else:
178         job.addFrame(scene.frame_current)
179
180     filename = bpy.data.filepath
181     job.addFile(filename)
182
183     job_name = netsettings.job_name
184     path, name = os.path.split(filename)
185     if job_name == "[default]":
186         job_name = name
187
188     ###########################
189     # LIBRARIES
190     ###########################
191     for lib in bpy.data.libraries:
192         file_path = bpy.path.abspath(lib.filepath)
193         if os.path.exists(file_path):
194             job.addFile(file_path)
195
196     ###########################
197     # IMAGES
198     ###########################
199     for image in bpy.data.images:
200         if image.source == "FILE" and not image.packed_file:
201             file_path = bpy.path.abspath(image.filepath)
202             if os.path.exists(file_path):
203                 job.addFile(file_path)
204                 
205                 tex_path = os.path.splitext(file_path)[0] + ".tex"
206                 if os.path.exists(tex_path):
207                     job.addFile(tex_path)
208
209     ###########################
210     # FLUID + POINT CACHE
211     ###########################
212     root, ext = os.path.splitext(name)
213     default_path = path + os.sep + "blendcache_" + root + os.sep # need an API call for that
214
215     for object in bpy.data.objects:
216         for modifier in object.modifiers:
217             if modifier.type == 'FLUID_SIMULATION' and modifier.settings.type == "DOMAIN":
218                 addFluidFiles(job, bpy.path.abspath(modifier.settings.path))
219             elif modifier.type == "CLOTH":
220                 addPointCache(job, object, modifier.point_cache, default_path)
221             elif modifier.type == "SOFT_BODY":
222                 addPointCache(job, object, modifier.point_cache, default_path)
223             elif modifier.type == "SMOKE" and modifier.smoke_type == "TYPE_DOMAIN":
224                 addPointCache(job, object, modifier.domain_settings.point_cache, default_path)
225             elif modifier.type == "MULTIRES" and modifier.is_external:
226                 file_path = bpy.path.abspath(modifier.filepath)
227                 job.addFile(file_path)
228
229         # particles modifier are stupid and don't contain data
230         # we have to go through the object property
231         for psys in object.particle_systems:
232             addPointCache(job, object, psys.point_cache, default_path)
233
234     #print(job.files)
235
236     fillCommonJobSettings(job, job_name, netsettings)
237
238     # try to send path first
239     conn.request("POST", "/job", json.dumps(job.serialize()))
240     response = conn.getresponse()
241     response.read()
242
243     job_id = response.getheader("job-id")
244
245     # if not ACCEPTED (but not processed), send files
246     if response.status == http.client.ACCEPTED:
247         for rfile in job.files:
248             f = open(rfile.filepath, "rb")
249             conn.request("PUT", fileURL(job_id, rfile.index), f)
250             f.close()
251             response = conn.getresponse()
252             response.read()
253
254     # server will reply with ACCEPTED until all files are found
255
256     return job_id
257
258 def requestResult(conn, job_id, frame):
259     conn.request("GET", renderURL(job_id, frame))
260
261 class NetworkRenderEngine(bpy.types.RenderEngine):
262     bl_idname = 'NET_RENDER'
263     bl_label = "Network Render"
264     bl_postprocess = False
265     def render(self, scene):
266         if scene.network_render.mode == "RENDER_CLIENT":
267             self.render_client(scene)
268         elif scene.network_render.mode == "RENDER_SLAVE":
269             self.render_slave(scene)
270         elif scene.network_render.mode == "RENDER_MASTER":
271             self.render_master(scene)
272         else:
273             print("UNKNOWN OPERATION MODE")
274
275     def render_master(self, scene):
276         netsettings = scene.network_render
277
278         address = "" if netsettings.server_address == "[default]" else netsettings.server_address
279
280         master.runMaster((address, netsettings.server_port), netsettings.use_master_broadcast, netsettings.use_master_clear, bpy.path.abspath(netsettings.path), self.update_stats, self.test_break)
281
282
283     def render_slave(self, scene):
284         slave.render_slave(self, scene.network_render, scene.render.threads)
285
286     def render_client(self, scene):
287         netsettings = scene.network_render
288         self.update_stats("", "Network render client initiation")
289
290
291         conn = clientConnection(netsettings.server_address, netsettings.server_port)
292
293         if conn:
294             # Sending file
295
296             self.update_stats("", "Network render exporting")
297
298             new_job = False
299
300             job_id = netsettings.job_id
301
302             # reading back result
303
304             self.update_stats("", "Network render waiting for results")
305             
306              
307             requestResult(conn, job_id, scene.frame_current)
308             response = conn.getresponse()
309             buf = response.read()
310
311             if response.status == http.client.NO_CONTENT:
312                 new_job = True
313                 netsettings.job_id = clientSendJob(conn, scene)
314                 job_id = netsettings.job_id
315
316                 requestResult(conn, job_id, scene.frame_current)
317                 response = conn.getresponse()
318                 buf = response.read()
319                 
320             while response.status == http.client.ACCEPTED and not self.test_break():
321                 time.sleep(1)
322                 requestResult(conn, job_id, scene.frame_current)
323                 response = conn.getresponse()
324                 buf = response.read()
325
326             # cancel new jobs (animate on network) on break
327             if self.test_break() and new_job:
328                 conn.request("POST", cancelURL(job_id))
329                 response = conn.getresponse()
330                 response.read()
331                 print( response.status, response.reason )
332                 netsettings.job_id = 0
333
334             if response.status != http.client.OK:
335                 conn.close()
336                 return
337
338             r = scene.render
339             x= int(r.resolution_x*r.resolution_percentage*0.01)
340             y= int(r.resolution_y*r.resolution_percentage*0.01)
341             
342             result_path = os.path.join(bpy.path.abspath(netsettings.path), "output.exr")
343             
344             folder = os.path.split(result_path)[0]
345             
346             if not os.path.exists(folder):
347                 os.mkdir(folder)
348
349             f = open(result_path, "wb")
350
351             f.write(buf)
352
353             f.close()
354
355             result = self.begin_result(0, 0, x, y)
356             result.load_from_file(result_path)
357             self.end_result(result)
358
359             conn.close()
360
361 def compatible(module):
362     module = __import__(module)
363     for subclass in module.__dict__.values():
364         try:            subclass.COMPAT_ENGINES.add('NET_RENDER')
365         except: pass
366     del module
367
368 #compatible("properties_render")
369 compatible("properties_world")
370 compatible("properties_material")
371 compatible("properties_data_mesh")
372 compatible("properties_data_camera")
373 compatible("properties_texture")