release/scripts/io/netrender/client.py

   1 # ##### BEGIN GPL LICENSE BLOCK #####
   2 #
   3 #  This program is free software; you can redistribute it and/or
   4 #  modify it under the terms of the GNU General Public License
   5 #  as published by the Free Software Foundation; either version 2
   6 #  of the License, or (at your option) any later version.
   7 #
   8 #  This program is distributed in the hope that it will be useful,
   9 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 #  GNU General Public License for more details.
  12 #
  13 #  You should have received a copy of the GNU General Public License
  14 #  along with this program; if not, write to the Free Software Foundation,
  15 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16 #
  17 # ##### END GPL LICENSE BLOCK #####
  18
  19 import bpy
  20 import sys, os, re
  21 import http, http.client, http.server, urllib
  22 import subprocess, shutil, time, hashlib
  23 import json
  24
  25 import netrender
  26 import netrender.model
  27 import netrender.slave as slave
  28 import netrender.master as master
  29 from netrender.utils import *
  30
  31 def addFluidFiles(job, path):
  32     if os.path.exists(path):
  33         pattern = re.compile("fluidsurface_(final|preview)_([0-9]+)\.(bobj|bvel)\.gz")
  34
  35         for fluid_file in sorted(os.listdir(path)):
  36             match = pattern.match(fluid_file)
  37
  38             if match:
  39                 # fluid frames starts at 0, which explains the +1
  40                 # This is stupid
  41                 current_frame = int(match.groups()[1]) + 1
  42                 job.addFile(path + fluid_file, current_frame, current_frame)
  43
  44 def addPointCache(job, ob, point_cache, default_path):
  45     if not point_cache.use_disk_cache:
  46         return
  47
  48
  49     name = point_cache.name
  50     if name == "":
  51         name = "".join(["%02X" % ord(c) for c in ob.name])
  52
  53     cache_path = bpy.path.abspath(point_cache.filepath) if point_cache.use_external else default_path
  54
  55     index = "%02i" % point_cache.index
  56
  57     if os.path.exists(cache_path):
  58         pattern = re.compile(name + "_([0-9]+)_" + index + "\.bphys")
  59
  60         cache_files = []
  61
  62         for cache_file in sorted(os.listdir(cache_path)):
  63             match = pattern.match(cache_file)
  64
  65             if match:
  66                 cache_frame = int(match.groups()[0])
  67                 cache_files.append((cache_frame, cache_file))
  68
  69         cache_files.sort()
  70
  71         if len(cache_files) == 1:
  72             cache_frame, cache_file = cache_files[0]
  73             job.addFile(cache_path + cache_file, cache_frame, cache_frame)
  74         else:
  75             for i in range(len(cache_files)):
  76                 current_item = cache_files[i]
  77                 next_item = cache_files[i+1] if i + 1 < len(cache_files) else None
  78                 previous_item = cache_files[i - 1] if i > 0 else None
  79
  80                 current_frame, current_file = current_item
  81
  82                 if  not next_item and not previous_item:
  83                     job.addFile(cache_path + current_file, current_frame, current_frame)
  84                 elif next_item and not previous_item:
  85                     next_frame = next_item[0]
  86                     job.addFile(cache_path + current_file, current_frame, next_frame - 1)
  87                 elif not next_item and previous_item:
  88                     previous_frame = previous_item[0]
  89                     job.addFile(cache_path + current_file, previous_frame + 1, current_frame)
  90                 else:
  91                     next_frame = next_item[0]
  92                     previous_frame = previous_item[0]
  93                     job.addFile(cache_path + current_file, previous_frame + 1, next_frame - 1)
  94
  95 def fillCommonJobSettings(job, job_name, netsettings):
  96     job.name = job_name
  97     job.category = netsettings.job_category
  98
  99     for slave in netrender.blacklist:
 100         job.blacklist.append(slave.id)
 101
 102     job.chunks = netsettings.chunks
 103     job.priority = netsettings.priority
 104
 105     if netsettings.job_type == "JOB_BLENDER":
 106         job.type = netrender.model.JOB_BLENDER
 107     elif netsettings.job_type == "JOB_PROCESS":
 108         job.type = netrender.model.JOB_PROCESS
 109     elif netsettings.job_type == "JOB_VCS":
 110         job.type = netrender.model.JOB_VCS
 111
 112 def clientSendJob(conn, scene, anim = False):
 113     netsettings = scene.network_render
 114     if netsettings.job_type == "JOB_BLENDER":
 115         return clientSendJobBlender(conn, scene, anim)
 116     elif netsettings.job_type == "JOB_VCS":
 117         return clientSendJobVCS(conn, scene, anim)
 118
 119 def clientSendJobVCS(conn, scene, anim = False):
 120     netsettings = scene.network_render
 121     job = netrender.model.RenderJob()
 122
 123     if anim:
 124         for f in range(scene.frame_start, scene.frame_end + 1):
 125             job.addFrame(f)
 126     else:
 127         job.addFrame(scene.frame_current)
 128
 129     filename = bpy.data.filepath
 130
 131     if not filename.startswith(netsettings.vcs_wpath):
 132         # this is an error, need better way to handle this
 133         return
 134
 135     filename = filename[len(netsettings.vcs_wpath):]
 136
 137     if filename[0] in (os.sep, os.altsep):
 138         filename = filename[1:]
 139
 140     print("CREATING VCS JOB", filename)
 141
 142     job.addFile(filename, signed=False)
 143
 144     job_name = netsettings.job_name
 145     path, name = os.path.split(filename)
 146     if job_name == "[default]":
 147         job_name = name
 148
 149
 150     fillCommonJobSettings(job, job_name, netsettings)
 151
 152     # VCS Specific code
 153     job.version_info = netrender.model.VersioningInfo()
 154     job.version_info.system = netsettings.vcs_system
 155     job.version_info.wpath = netsettings.vcs_wpath
 156     job.version_info.rpath = netsettings.vcs_rpath
 157     job.version_info.revision = netsettings.vcs_revision
 158
 159     # try to send path first
 160     conn.request("POST", "/job", json.dumps(job.serialize()))
 161     response = conn.getresponse()
 162     response.read()
 163
 164     job_id = response.getheader("job-id")
 165
 166     # a VCS job is always good right now, need error handling
 167
 168     return job_id
 169
 170 def clientSendJobBlender(conn, scene, anim = False):
 171     netsettings = scene.network_render
 172     job = netrender.model.RenderJob()
 173
 174     if anim:
 175         for f in range(scene.frame_start, scene.frame_end + 1):
 176             job.addFrame(f)
 177     else:
 178         job.addFrame(scene.frame_current)
 179
 180     filename = bpy.data.filepath
 181     job.addFile(filename)
 182
 183     job_name = netsettings.job_name
 184     path, name = os.path.split(filename)
 185     if job_name == "[default]":
 186         job_name = name
 187
 188     ###########################
 189     # LIBRARIES
 190     ###########################
 191     for lib in bpy.data.libraries:
 192         file_path = bpy.path.abspath(lib.filepath)
 193         if os.path.exists(file_path):
 194             job.addFile(file_path)
 195
 196     ###########################
 197     # IMAGES
 198     ###########################
 199     for image in bpy.data.images:
 200         if image.source == "FILE" and not image.packed_file:
 201             file_path = bpy.path.abspath(image.filepath)
 202             if os.path.exists(file_path):
 203                 job.addFile(file_path)
 204
 205                 tex_path = os.path.splitext(file_path)[0] + ".tex"
 206                 if os.path.exists(tex_path):
 207                     job.addFile(tex_path)
 208
 209     ###########################
 210     # FLUID + POINT CACHE
 211     ###########################
 212     root, ext = os.path.splitext(name)
 213     default_path = path + os.sep + "blendcache_" + root + os.sep # need an API call for that
 214
 215     for object in bpy.data.objects:
 216         for modifier in object.modifiers:
 217             if modifier.type == 'FLUID_SIMULATION' and modifier.settings.type == "DOMAIN":
 218                 addFluidFiles(job, bpy.path.abspath(modifier.settings.path))
 219             elif modifier.type == "CLOTH":
 220                 addPointCache(job, object, modifier.point_cache, default_path)
 221             elif modifier.type == "SOFT_BODY":
 222                 addPointCache(job, object, modifier.point_cache, default_path)
 223             elif modifier.type == "SMOKE" and modifier.smoke_type == "TYPE_DOMAIN":
 224                 addPointCache(job, object, modifier.domain_settings.point_cache, default_path)
 225             elif modifier.type == "MULTIRES" and modifier.is_external:
 226                 file_path = bpy.path.abspath(modifier.filepath)
 227                 job.addFile(file_path)
 228
 229         # particles modifier are stupid and don't contain data
 230         # we have to go through the object property
 231         for psys in object.particle_systems:
 232             addPointCache(job, object, psys.point_cache, default_path)
 233
 234     #print(job.files)
 235
 236     fillCommonJobSettings(job, job_name, netsettings)
 237
 238     # try to send path first
 239     conn.request("POST", "/job", json.dumps(job.serialize()))
 240     response = conn.getresponse()
 241     response.read()
 242
 243     job_id = response.getheader("job-id")
 244
 245     # if not ACCEPTED (but not processed), send files
 246     if response.status == http.client.ACCEPTED:
 247         for rfile in job.files:
 248             f = open(rfile.filepath, "rb")
 249             conn.request("PUT", fileURL(job_id, rfile.index), f)
 250             f.close()
 251             response = conn.getresponse()
 252             response.read()
 253
 254     # server will reply with ACCEPTED until all files are found
 255
 256     return job_id
 257
 258 def requestResult(conn, job_id, frame):
 259     conn.request("GET", renderURL(job_id, frame))
 260
 261 class NetworkRenderEngine(bpy.types.RenderEngine):
 262     bl_idname = 'NET_RENDER'
 263     bl_label = "Network Render"
 264     bl_postprocess = False
 265     def render(self, scene):
 266         if scene.network_render.mode == "RENDER_CLIENT":
 267             self.render_client(scene)
 268         elif scene.network_render.mode == "RENDER_SLAVE":
 269             self.render_slave(scene)
 270         elif scene.network_render.mode == "RENDER_MASTER":
 271             self.render_master(scene)
 272         else:
 273             print("UNKNOWN OPERATION MODE")
 274
 275     def render_master(self, scene):
 276         netsettings = scene.network_render
 277
 278         address = "" if netsettings.server_address == "[default]" else netsettings.server_address
 279
 280         master.runMaster((address, netsettings.server_port), netsettings.use_master_broadcast, netsettings.use_master_clear, bpy.path.abspath(netsettings.path), self.update_stats, self.test_break)
 281
 282
 283     def render_slave(self, scene):
 284         slave.render_slave(self, scene.network_render, scene.render.threads)
 285
 286     def render_client(self, scene):
 287         netsettings = scene.network_render
 288         self.update_stats("", "Network render client initiation")
 289
 290
 291         conn = clientConnection(netsettings.server_address, netsettings.server_port)
 292
 293         if conn:
 294             # Sending file
 295
 296             self.update_stats("", "Network render exporting")
 297
 298             new_job = False
 299
 300             job_id = netsettings.job_id
 301
 302             # reading back result
 303
 304             self.update_stats("", "Network render waiting for results")
 305
 306
 307             requestResult(conn, job_id, scene.frame_current)
 308             response = conn.getresponse()
 309             buf = response.read()
 310
 311             if response.status == http.client.NO_CONTENT:
 312                 new_job = True
 313                 netsettings.job_id = clientSendJob(conn, scene)
 314                 job_id = netsettings.job_id
 315
 316                 requestResult(conn, job_id, scene.frame_current)
 317                 response = conn.getresponse()
 318                 buf = response.read()
 319
 320             while response.status == http.client.ACCEPTED and not self.test_break():
 321                 time.sleep(1)
 322                 requestResult(conn, job_id, scene.frame_current)
 323                 response = conn.getresponse()
 324                 buf = response.read()
 325
 326             # cancel new jobs (animate on network) on break
 327             if self.test_break() and new_job:
 328                 conn.request("POST", cancelURL(job_id))
 329                 response = conn.getresponse()
 330                 response.read()
 331                 print( response.status, response.reason )
 332                 netsettings.job_id = 0
 333
 334             if response.status != http.client.OK:
 335                 conn.close()
 336                 return
 337
 338             r = scene.render
 339             x= int(r.resolution_x*r.resolution_percentage*0.01)
 340             y= int(r.resolution_y*r.resolution_percentage*0.01)
 341
 342             result_path = os.path.join(bpy.path.abspath(netsettings.path), "output.exr")
 343
 344             folder = os.path.split(result_path)[0]
 345
 346             if not os.path.exists(folder):
 347                 os.mkdir(folder)
 348
 349             f = open(result_path, "wb")
 350
 351             f.write(buf)
 352
 353             f.close()
 354
 355             result = self.begin_result(0, 0, x, y)
 356             result.load_from_file(result_path)
 357             self.end_result(result)
 358
 359             conn.close()
 360
 361 def compatible(module):
 362     module = __import__(module)
 363     for subclass in module.__dict__.values():
 364         try:            subclass.COMPAT_ENGINES.add('NET_RENDER')
 365         except: pass
 366     del module
 367
 368 #compatible("properties_render")
 369 compatible("properties_world")
 370 compatible("properties_material")
 371 compatible("properties_data_mesh")
 372 compatible("properties_data_camera")
 373 compatible("properties_texture")