Merged revision(s) 58994-59032 from trunk/blender into soc-2013-dingto.
[blender.git] / intern / cycles / device / device_cuda.cpp
1 /*
2  * Copyright 2011, Blender Foundation.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  */
18
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22
23 #include "device.h"
24 #include "device_intern.h"
25
26 #include "buffers.h"
27
28 #include "util_cuda.h"
29 #include "util_debug.h"
30 #include "util_map.h"
31 #include "util_opengl.h"
32 #include "util_path.h"
33 #include "util_system.h"
34 #include "util_types.h"
35 #include "util_time.h"
36
37 CCL_NAMESPACE_BEGIN
38
39 class CUDADevice : public Device
40 {
41 public:
42         TaskPool task_pool;
43         CUdevice cuDevice;
44         CUcontext cuContext;
45         CUmodule cuModule;
46         map<device_ptr, bool> tex_interp_map;
47         int cuDevId;
48         bool first_error;
49
50         struct PixelMem {
51                 GLuint cuPBO;
52                 CUgraphicsResource cuPBOresource;
53                 GLuint cuTexId;
54                 int w, h;
55         };
56
57         map<device_ptr, PixelMem> pixel_mem_map;
58
59         CUdeviceptr cuda_device_ptr(device_ptr mem)
60         {
61                 return (CUdeviceptr)mem;
62         }
63
64         static const char *cuda_error_string(CUresult result)
65         {
66                 switch(result) {
67                         case CUDA_SUCCESS: return "No errors";
68                         case CUDA_ERROR_INVALID_VALUE: return "Invalid value";
69                         case CUDA_ERROR_OUT_OF_MEMORY: return "Out of memory";
70                         case CUDA_ERROR_NOT_INITIALIZED: return "Driver not initialized";
71                         case CUDA_ERROR_DEINITIALIZED: return "Driver deinitialized";
72
73                         case CUDA_ERROR_NO_DEVICE: return "No CUDA-capable device available";
74                         case CUDA_ERROR_INVALID_DEVICE: return "Invalid device";
75
76                         case CUDA_ERROR_INVALID_IMAGE: return "Invalid kernel image";
77                         case CUDA_ERROR_INVALID_CONTEXT: return "Invalid context";
78                         case CUDA_ERROR_CONTEXT_ALREADY_CURRENT: return "Context already current";
79                         case CUDA_ERROR_MAP_FAILED: return "Map failed";
80                         case CUDA_ERROR_UNMAP_FAILED: return "Unmap failed";
81                         case CUDA_ERROR_ARRAY_IS_MAPPED: return "Array is mapped";
82                         case CUDA_ERROR_ALREADY_MAPPED: return "Already mapped";
83                         case CUDA_ERROR_NO_BINARY_FOR_GPU: return "No binary for GPU";
84                         case CUDA_ERROR_ALREADY_ACQUIRED: return "Already acquired";
85                         case CUDA_ERROR_NOT_MAPPED: return "Not mapped";
86                         case CUDA_ERROR_NOT_MAPPED_AS_ARRAY: return "Mapped resource not available for access as an array";
87                         case CUDA_ERROR_NOT_MAPPED_AS_POINTER: return "Mapped resource not available for access as a pointer";
88                         case CUDA_ERROR_ECC_UNCORRECTABLE: return "Uncorrectable ECC error detected";
89                         case CUDA_ERROR_UNSUPPORTED_LIMIT: return "CUlimit not supported by device";
90
91                         case CUDA_ERROR_INVALID_SOURCE: return "Invalid source";
92                         case CUDA_ERROR_FILE_NOT_FOUND: return "File not found";
93                         case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND: return "Link to a shared object failed to resolve";
94                         case CUDA_ERROR_SHARED_OBJECT_INIT_FAILED: return "Shared object initialization failed";
95
96                         case CUDA_ERROR_INVALID_HANDLE: return "Invalid handle";
97
98                         case CUDA_ERROR_NOT_FOUND: return "Not found";
99
100                         case CUDA_ERROR_NOT_READY: return "CUDA not ready";
101
102                         case CUDA_ERROR_LAUNCH_FAILED: return "Launch failed";
103                         case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES: return "Launch exceeded resources";
104                         case CUDA_ERROR_LAUNCH_TIMEOUT: return "Launch exceeded timeout";
105                         case CUDA_ERROR_LAUNCH_INCOMPATIBLE_TEXTURING: return "Launch with incompatible texturing";
106
107                         case CUDA_ERROR_UNKNOWN: return "Unknown error";
108
109                         default: return "Unknown CUDA error value";
110                 }
111         }
112
113 /*#ifdef NDEBUG
114 #define cuda_abort()
115 #else
116 #define cuda_abort() abort()
117 #endif*/
118         void cuda_error_documentation()
119         {
120                 if(first_error) {
121                         fprintf(stderr, "\nRefer to the Cycles GPU rendering documentation for possible solutions:\n");
122                         fprintf(stderr, "http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/GPU_Rendering\n\n");
123                         first_error = false;
124                 }
125         }
126
127 #define cuda_assert(stmt) \
128         { \
129                 CUresult result = stmt; \
130                 \
131                 if(result != CUDA_SUCCESS) { \
132                         string message = string_printf("CUDA error: %s in %s", cuda_error_string(result), #stmt); \
133                         if(error_msg == "") \
134                                 error_msg = message; \
135                         fprintf(stderr, "%s\n", message.c_str()); \
136                         /*cuda_abort();*/ \
137                         cuda_error_documentation(); \
138                 } \
139         }
140
141         bool cuda_error_(CUresult result, const string& stmt)
142         {
143                 if(result == CUDA_SUCCESS)
144                         return false;
145
146                 string message = string_printf("CUDA error at %s: %s", stmt.c_str(), cuda_error_string(result));
147                 if(error_msg == "")
148                         error_msg = message;
149                 fprintf(stderr, "%s\n", message.c_str());
150                 cuda_error_documentation();
151                 return true;
152         }
153
154 #define cuda_error(stmt) cuda_error_(stmt, #stmt)
155
156         void cuda_error_message(const string& message)
157         {
158                 if(error_msg == "")
159                         error_msg = message;
160                 fprintf(stderr, "%s\n", message.c_str());
161                 cuda_error_documentation();
162         }
163
164         void cuda_push_context()
165         {
166                 cuda_assert(cuCtxSetCurrent(cuContext))
167         }
168
169         void cuda_pop_context()
170         {
171                 cuda_assert(cuCtxSetCurrent(NULL));
172         }
173
174         CUDADevice(DeviceInfo& info, Stats &stats, bool background_) : Device(stats)
175         {
176                 first_error = true;
177                 background = background_;
178
179                 cuDevId = info.num;
180                 cuDevice = 0;
181                 cuContext = 0;
182
183                 /* intialize */
184                 if(cuda_error(cuInit(0)))
185                         return;
186
187                 /* setup device and context */
188                 if(cuda_error(cuDeviceGet(&cuDevice, cuDevId)))
189                         return;
190
191                 CUresult result;
192
193                 if(background) {
194                         result = cuCtxCreate(&cuContext, 0, cuDevice);
195                 }
196                 else {
197                         result = cuGLCtxCreate(&cuContext, 0, cuDevice);
198
199                         if(result != CUDA_SUCCESS) {
200                                 result = cuCtxCreate(&cuContext, 0, cuDevice);
201                                 background = true;
202                         }
203                 }
204
205                 if(cuda_error_(result, "cuCtxCreate"))
206                         return;
207
208                 cuda_pop_context();
209         }
210
211         ~CUDADevice()
212         {
213                 task_pool.stop();
214
215                 cuda_push_context();
216                 cuda_assert(cuCtxDetach(cuContext))
217         }
218
219         bool support_device(bool experimental)
220         {
221                 if(!experimental) {
222                         int major, minor;
223                         cuDeviceComputeCapability(&major, &minor, cuDevId);
224
225                         if(major < 2) {
226                                 cuda_error_message(string_printf("CUDA device supported only with compute capability 2.0 or up, found %d.%d.", major, minor));
227                                 return false;
228                         }
229                 }
230
231                 return true;
232         }
233
234         string compile_kernel()
235         {
236                 /* compute cubin name */
237                 int major, minor;
238                 cuDeviceComputeCapability(&major, &minor, cuDevId);
239
240                 /* attempt to use kernel provided with blender */
241                 string cubin = path_get(string_printf("lib/kernel_sm_%d%d.cubin", major, minor));
242                 if(path_exists(cubin))
243                         return cubin;
244
245                 /* not found, try to use locally compiled kernel */
246                 string kernel_path = path_get("kernel");
247                 string md5 = path_files_md5_hash(kernel_path);
248
249                 cubin = string_printf("cycles_kernel_sm%d%d_%s.cubin", major, minor, md5.c_str());
250                 cubin = path_user_get(path_join("cache", cubin));
251
252                 /* if exists already, use it */
253                 if(path_exists(cubin))
254                         return cubin;
255
256 #ifdef _WIN32
257                 if(cuHavePrecompiledKernels()) {
258                         if(major < 2)
259                                 cuda_error_message(string_printf("CUDA device requires compute capability 2.0 or up, found %d.%d. Your GPU is not supported.", major, minor));
260                         else
261                                 cuda_error_message(string_printf("CUDA binary kernel for this graphics card compute capability (%d.%d) not found.", major, minor));
262                         return "";
263                 }
264 #endif
265
266                 /* if not, find CUDA compiler */
267                 string nvcc = cuCompilerPath();
268
269                 if(nvcc == "") {
270                         cuda_error_message("CUDA nvcc compiler not found. Install CUDA toolkit in default location.");
271                         return "";
272                 }
273
274                 int cuda_version = cuCompilerVersion();
275
276                 if(cuda_version == 0) {
277                         cuda_error_message("CUDA nvcc compiler version could not be parsed.");
278                         return "";
279                 }
280
281                 if(cuda_version != 50)
282                         printf("CUDA version %d.%d detected, build may succeed but only CUDA 5.0 is officially supported.\n", cuda_version/10, cuda_version%10);
283
284                 /* compile */
285                 string kernel = path_join(kernel_path, "kernel.cu");
286                 string include = kernel_path;
287                 const int machine = system_cpu_bits();
288                 string arch_flags;
289
290                 /* build flags depending on CUDA version and arch */
291                 if(cuda_version < 50) {
292                         /* CUDA 4.x */
293                         if(major == 1) {
294                                 /* sm_1x */
295                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0";
296                         }
297                         else if(major == 2) {
298                                 /* sm_2x */
299                                 arch_flags = "--maxrregcount=24";
300                         }
301                         else {
302                                 /* sm_3x */
303                                 arch_flags = "--maxrregcount=32";
304                         }
305                 }
306                 else {
307                         /* CUDA 5.x */
308                         if(major == 1) {
309                                 /* sm_1x */
310                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0 --use_fast_math";
311                         }
312                         else if(major == 2) {
313                                 /* sm_2x */
314                                 arch_flags = "--maxrregcount=32 --use_fast_math";
315                         }
316                         else {
317                                 /* sm_3x */
318                                 arch_flags = "--maxrregcount=32 --use_fast_math";
319                         }
320                 }
321
322                 double starttime = time_dt();
323                 printf("Compiling CUDA kernel ...\n");
324
325                 path_create_directories(cubin);
326
327                 string command = string_printf("\"%s\" -arch=sm_%d%d -m%d --cubin \"%s\" "
328                         "-o \"%s\" --ptxas-options=\"-v\" %s -I\"%s\" -DNVCC -D__KERNEL_CUDA_VERSION__=%d",
329                         nvcc.c_str(), major, minor, machine, kernel.c_str(), cubin.c_str(), arch_flags.c_str(), include.c_str(), cuda_version);
330
331                 printf("%s\n", command.c_str());
332
333                 if(system(command.c_str()) == -1) {
334                         cuda_error_message("Failed to execute compilation command, see console for details.");
335                         return "";
336                 }
337
338                 /* verify if compilation succeeded */
339                 if(!path_exists(cubin)) {
340                         cuda_error_message("CUDA kernel compilation failed, see console for details.");
341                         return "";
342                 }
343
344                 printf("Kernel compilation finished in %.2lfs.\n", time_dt() - starttime);
345
346                 return cubin;
347         }
348
349         bool load_kernels(bool experimental)
350         {
351                 /* check if cuda init succeeded */
352                 if(cuContext == 0)
353                         return false;
354                 
355                 /* check if GPU is supported with current feature set */
356                 if(!support_device(experimental))
357                         return false;
358
359                 /* get kernel */
360                 string cubin = compile_kernel();
361
362                 if(cubin == "")
363                         return false;
364
365                 /* open module */
366                 cuda_push_context();
367
368                 CUresult result = cuModuleLoad(&cuModule, cubin.c_str());
369                 if(cuda_error_(result, "cuModuleLoad"))
370                         cuda_error_message(string_printf("Failed loading CUDA kernel %s.", cubin.c_str()));
371
372                 cuda_pop_context();
373
374                 return (result == CUDA_SUCCESS);
375         }
376
377         void mem_alloc(device_memory& mem, MemoryType type)
378         {
379                 cuda_push_context();
380                 CUdeviceptr device_pointer;
381                 size_t size = mem.memory_size();
382                 cuda_assert(cuMemAlloc(&device_pointer, size))
383                 mem.device_pointer = (device_ptr)device_pointer;
384                 stats.mem_alloc(size);
385                 cuda_pop_context();
386         }
387
388         void mem_copy_to(device_memory& mem)
389         {
390                 cuda_push_context();
391                 if(mem.device_pointer)
392                         cuda_assert(cuMemcpyHtoD(cuda_device_ptr(mem.device_pointer), (void*)mem.data_pointer, mem.memory_size()))
393                 cuda_pop_context();
394         }
395
396         void mem_copy_from(device_memory& mem, int y, int w, int h, int elem)
397         {
398                 size_t offset = elem*y*w;
399                 size_t size = elem*w*h;
400
401                 cuda_push_context();
402                 if(mem.device_pointer) {
403                         cuda_assert(cuMemcpyDtoH((uchar*)mem.data_pointer + offset,
404                                 (CUdeviceptr)((uchar*)mem.device_pointer + offset), size))
405                 }
406                 else {
407                         memset((char*)mem.data_pointer + offset, 0, size);
408                 }
409                 cuda_pop_context();
410         }
411
412         void mem_zero(device_memory& mem)
413         {
414                 memset((void*)mem.data_pointer, 0, mem.memory_size());
415
416                 cuda_push_context();
417                 if(mem.device_pointer)
418                         cuda_assert(cuMemsetD8(cuda_device_ptr(mem.device_pointer), 0, mem.memory_size()))
419                 cuda_pop_context();
420         }
421
422         void mem_free(device_memory& mem)
423         {
424                 if(mem.device_pointer) {
425                         cuda_push_context();
426                         cuda_assert(cuMemFree(cuda_device_ptr(mem.device_pointer)))
427                         cuda_pop_context();
428
429                         mem.device_pointer = 0;
430
431                         stats.mem_free(mem.memory_size());
432                 }
433         }
434
435         void const_copy_to(const char *name, void *host, size_t size)
436         {
437                 CUdeviceptr mem;
438                 size_t bytes;
439
440                 cuda_push_context();
441                 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule, name))
442                 //assert(bytes == size);
443                 cuda_assert(cuMemcpyHtoD(mem, host, size))
444                 cuda_pop_context();
445         }
446
447         void tex_alloc(const char *name, device_memory& mem, bool interpolation, bool periodic)
448         {
449                 /* determine format */
450                 CUarray_format_enum format;
451                 size_t dsize = datatype_size(mem.data_type);
452                 size_t size = mem.memory_size();
453
454                 switch(mem.data_type) {
455                         case TYPE_UCHAR: format = CU_AD_FORMAT_UNSIGNED_INT8; break;
456                         case TYPE_UINT: format = CU_AD_FORMAT_UNSIGNED_INT32; break;
457                         case TYPE_INT: format = CU_AD_FORMAT_SIGNED_INT32; break;
458                         case TYPE_FLOAT: format = CU_AD_FORMAT_FLOAT; break;
459                         default: assert(0); return;
460                 }
461
462                 CUtexref texref = NULL;
463
464                 cuda_push_context();
465                 cuda_assert(cuModuleGetTexRef(&texref, cuModule, name))
466
467                 if(!texref) {
468                         cuda_pop_context();
469                         return;
470                 }
471
472                 if(interpolation) {
473                         CUarray handle = NULL;
474                         CUDA_ARRAY_DESCRIPTOR desc;
475
476                         desc.Width = mem.data_width;
477                         desc.Height = mem.data_height;
478                         desc.Format = format;
479                         desc.NumChannels = mem.data_elements;
480
481                         cuda_assert(cuArrayCreate(&handle, &desc))
482
483                         if(!handle) {
484                                 cuda_pop_context();
485                                 return;
486                         }
487
488                         if(mem.data_height > 1) {
489                                 CUDA_MEMCPY2D param;
490                                 memset(&param, 0, sizeof(param));
491                                 param.dstMemoryType = CU_MEMORYTYPE_ARRAY;
492                                 param.dstArray = handle;
493                                 param.srcMemoryType = CU_MEMORYTYPE_HOST;
494                                 param.srcHost = (void*)mem.data_pointer;
495                                 param.srcPitch = mem.data_width*dsize*mem.data_elements;
496                                 param.WidthInBytes = param.srcPitch;
497                                 param.Height = mem.data_height;
498
499                                 cuda_assert(cuMemcpy2D(&param))
500                         }
501                         else
502                                 cuda_assert(cuMemcpyHtoA(handle, 0, (void*)mem.data_pointer, size))
503
504                         cuda_assert(cuTexRefSetArray(texref, handle, CU_TRSA_OVERRIDE_FORMAT))
505
506                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_LINEAR))
507                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_NORMALIZED_COORDINATES))
508
509                         mem.device_pointer = (device_ptr)handle;
510
511                         stats.mem_alloc(size);
512                 }
513                 else {
514                         cuda_pop_context();
515
516                         mem_alloc(mem, MEM_READ_ONLY);
517                         mem_copy_to(mem);
518
519                         cuda_push_context();
520
521                         cuda_assert(cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size))
522                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_POINT))
523                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_READ_AS_INTEGER))
524                 }
525
526                 if(periodic) {
527                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_WRAP))
528                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_WRAP))
529                 }
530                 else {
531                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_CLAMP))
532                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_CLAMP))
533                 }
534                 cuda_assert(cuTexRefSetFormat(texref, format, mem.data_elements))
535
536                 cuda_pop_context();
537
538                 tex_interp_map[mem.device_pointer] = interpolation;
539         }
540
541         void tex_free(device_memory& mem)
542         {
543                 if(mem.device_pointer) {
544                         if(tex_interp_map[mem.device_pointer]) {
545                                 cuda_push_context();
546                                 cuArrayDestroy((CUarray)mem.device_pointer);
547                                 cuda_pop_context();
548
549                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
550                                 mem.device_pointer = 0;
551
552                                 stats.mem_free(mem.memory_size());
553                         }
554                         else {
555                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
556                                 mem_free(mem);
557                         }
558                 }
559         }
560
561         void path_trace(RenderTile& rtile, int sample, bool progressive)
562         {
563                 if(have_error())
564                         return;
565
566                 cuda_push_context();
567
568                 CUfunction cuPathTrace;
569                 CUdeviceptr d_buffer = cuda_device_ptr(rtile.buffer);
570                 CUdeviceptr d_rng_state = cuda_device_ptr(rtile.rng_state);
571
572                 /* get kernel function */
573                 if(progressive)
574                         cuda_assert(cuModuleGetFunction(&cuPathTrace, cuModule, "kernel_cuda_path_trace_progressive"))
575                 else
576                         cuda_assert(cuModuleGetFunction(&cuPathTrace, cuModule, "kernel_cuda_path_trace_non_progressive"))
577                 
578                 /* pass in parameters */
579                 int offset = 0;
580                 
581                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_buffer, sizeof(d_buffer)))
582                 offset += sizeof(d_buffer);
583
584                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_rng_state, sizeof(d_rng_state)))
585                 offset += sizeof(d_rng_state);
586
587                 offset = align_up(offset, __alignof(sample));
588
589                 cuda_assert(cuParamSeti(cuPathTrace, offset, sample))
590                 offset += sizeof(sample);
591
592                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.x))
593                 offset += sizeof(rtile.x);
594
595                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.y))
596                 offset += sizeof(rtile.y);
597
598                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.w))
599                 offset += sizeof(rtile.w);
600
601                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.h))
602                 offset += sizeof(rtile.h);
603
604                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.offset))
605                 offset += sizeof(rtile.offset);
606
607                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.stride))
608                 offset += sizeof(rtile.stride);
609
610                 cuda_assert(cuParamSetSize(cuPathTrace, offset))
611
612                 /* launch kernel: todo find optimal size, cache config for fermi */
613                 int xthreads = 16;
614                 int ythreads = 16;
615                 int xblocks = (rtile.w + xthreads - 1)/xthreads;
616                 int yblocks = (rtile.h + ythreads - 1)/ythreads;
617
618                 cuda_assert(cuFuncSetCacheConfig(cuPathTrace, CU_FUNC_CACHE_PREFER_L1))
619                 cuda_assert(cuFuncSetBlockShape(cuPathTrace, xthreads, ythreads, 1))
620                 cuda_assert(cuLaunchGrid(cuPathTrace, xblocks, yblocks))
621
622                 cuda_assert(cuCtxSynchronize())
623
624                 cuda_pop_context();
625         }
626
627         void tonemap(DeviceTask& task, device_ptr buffer, device_ptr rgba)
628         {
629                 if(have_error())
630                         return;
631
632                 cuda_push_context();
633
634                 CUfunction cuFilmConvert;
635                 CUdeviceptr d_rgba = map_pixels(rgba);
636                 CUdeviceptr d_buffer = cuda_device_ptr(buffer);
637
638                 /* get kernel function */
639                 cuda_assert(cuModuleGetFunction(&cuFilmConvert, cuModule, "kernel_cuda_tonemap"))
640
641                 /* pass in parameters */
642                 int offset = 0;
643
644                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_rgba, sizeof(d_rgba)))
645                 offset += sizeof(d_rgba);
646                 
647                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_buffer, sizeof(d_buffer)))
648                 offset += sizeof(d_buffer);
649
650                 int sample = task.sample;
651                 offset = align_up(offset, __alignof(sample));
652
653                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.sample))
654                 offset += sizeof(task.sample);
655
656                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.x))
657                 offset += sizeof(task.x);
658
659                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.y))
660                 offset += sizeof(task.y);
661
662                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.w))
663                 offset += sizeof(task.w);
664
665                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.h))
666                 offset += sizeof(task.h);
667
668                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.offset))
669                 offset += sizeof(task.offset);
670
671                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.stride))
672                 offset += sizeof(task.stride);
673
674                 cuda_assert(cuParamSetSize(cuFilmConvert, offset))
675
676                 /* launch kernel: todo find optimal size, cache config for fermi */
677                 int xthreads = 16;
678                 int ythreads = 16;
679                 int xblocks = (task.w + xthreads - 1)/xthreads;
680                 int yblocks = (task.h + ythreads - 1)/ythreads;
681
682                 cuda_assert(cuFuncSetCacheConfig(cuFilmConvert, CU_FUNC_CACHE_PREFER_L1))
683                 cuda_assert(cuFuncSetBlockShape(cuFilmConvert, xthreads, ythreads, 1))
684                 cuda_assert(cuLaunchGrid(cuFilmConvert, xblocks, yblocks))
685
686                 unmap_pixels(task.rgba);
687
688                 cuda_pop_context();
689         }
690
691         void shader(DeviceTask& task)
692         {
693                 if(have_error())
694                         return;
695
696                 cuda_push_context();
697
698                 CUfunction cuDisplace;
699                 CUdeviceptr d_input = cuda_device_ptr(task.shader_input);
700                 CUdeviceptr d_output = cuda_device_ptr(task.shader_output);
701
702                 /* get kernel function */
703                 cuda_assert(cuModuleGetFunction(&cuDisplace, cuModule, "kernel_cuda_shader"))
704                 
705                 /* pass in parameters */
706                 int offset = 0;
707                 
708                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_input, sizeof(d_input)))
709                 offset += sizeof(d_input);
710
711                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_output, sizeof(d_output)))
712                 offset += sizeof(d_output);
713
714                 int shader_eval_type = task.shader_eval_type;
715                 offset = align_up(offset, __alignof(shader_eval_type));
716
717                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_eval_type))
718                 offset += sizeof(task.shader_eval_type);
719
720                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_x))
721                 offset += sizeof(task.shader_x);
722
723                 cuda_assert(cuParamSetSize(cuDisplace, offset))
724
725                 /* launch kernel: todo find optimal size, cache config for fermi */
726                 int xthreads = 16;
727                 int xblocks = (task.shader_w + xthreads - 1)/xthreads;
728
729                 cuda_assert(cuFuncSetCacheConfig(cuDisplace, CU_FUNC_CACHE_PREFER_L1))
730                 cuda_assert(cuFuncSetBlockShape(cuDisplace, xthreads, 1, 1))
731                 cuda_assert(cuLaunchGrid(cuDisplace, xblocks, 1))
732
733                 cuda_pop_context();
734         }
735
736         CUdeviceptr map_pixels(device_ptr mem)
737         {
738                 if(!background) {
739                         PixelMem pmem = pixel_mem_map[mem];
740                         CUdeviceptr buffer;
741                         
742                         size_t bytes;
743                         cuda_assert(cuGraphicsMapResources(1, &pmem.cuPBOresource, 0))
744                         cuda_assert(cuGraphicsResourceGetMappedPointer(&buffer, &bytes, pmem.cuPBOresource))
745                         
746                         return buffer;
747                 }
748
749                 return cuda_device_ptr(mem);
750         }
751
752         void unmap_pixels(device_ptr mem)
753         {
754                 if(!background) {
755                         PixelMem pmem = pixel_mem_map[mem];
756
757                         cuda_assert(cuGraphicsUnmapResources(1, &pmem.cuPBOresource, 0))
758                 }
759         }
760
761         void pixels_alloc(device_memory& mem)
762         {
763                 if(!background) {
764                         PixelMem pmem;
765
766                         pmem.w = mem.data_width;
767                         pmem.h = mem.data_height;
768
769                         cuda_push_context();
770
771                         glGenBuffers(1, &pmem.cuPBO);
772                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
773                         glBufferData(GL_PIXEL_UNPACK_BUFFER, pmem.w*pmem.h*sizeof(GLfloat)*3, NULL, GL_DYNAMIC_DRAW);
774                         
775                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
776                         
777                         glGenTextures(1, &pmem.cuTexId);
778                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
779                         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, pmem.w, pmem.h, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
780                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
781                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
782                         glBindTexture(GL_TEXTURE_2D, 0);
783                         
784                         CUresult result = cuGraphicsGLRegisterBuffer(&pmem.cuPBOresource, pmem.cuPBO, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
785
786                         if(result == CUDA_SUCCESS) {
787                                 cuda_pop_context();
788
789                                 mem.device_pointer = pmem.cuTexId;
790                                 pixel_mem_map[mem.device_pointer] = pmem;
791
792                                 stats.mem_alloc(mem.memory_size());
793
794                                 return;
795                         }
796                         else {
797                                 /* failed to register buffer, fallback to no interop */
798                                 glDeleteBuffers(1, &pmem.cuPBO);
799                                 glDeleteTextures(1, &pmem.cuTexId);
800
801                                 cuda_pop_context();
802
803                                 background = true;
804                         }
805                 }
806
807                 Device::pixels_alloc(mem);
808         }
809
810         void pixels_copy_from(device_memory& mem, int y, int w, int h)
811         {
812                 if(!background) {
813                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
814
815                         cuda_push_context();
816
817                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
818                         uchar *pixels = (uchar*)glMapBuffer(GL_PIXEL_UNPACK_BUFFER, GL_READ_ONLY);
819                         size_t offset = sizeof(uchar)*4*y*w;
820                         memcpy((uchar*)mem.data_pointer + offset, pixels + offset, sizeof(uchar)*4*w*h);
821                         glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
822                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
823
824                         cuda_pop_context();
825
826                         return;
827                 }
828
829                 Device::pixels_copy_from(mem, y, w, h);
830         }
831
832         void pixels_free(device_memory& mem)
833         {
834                 if(mem.device_pointer) {
835                         if(!background) {
836                                 PixelMem pmem = pixel_mem_map[mem.device_pointer];
837
838                                 cuda_push_context();
839
840                                 cuda_assert(cuGraphicsUnregisterResource(pmem.cuPBOresource))
841                                 glDeleteBuffers(1, &pmem.cuPBO);
842                                 glDeleteTextures(1, &pmem.cuTexId);
843
844                                 cuda_pop_context();
845
846                                 pixel_mem_map.erase(pixel_mem_map.find(mem.device_pointer));
847                                 mem.device_pointer = 0;
848
849                                 stats.mem_free(mem.memory_size());
850
851                                 return;
852                         }
853
854                         Device::pixels_free(mem);
855                 }
856         }
857
858         void draw_pixels(device_memory& mem, int y, int w, int h, int dy, int width, int height, bool transparent)
859         {
860                 if(!background) {
861                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
862
863                         cuda_push_context();
864
865                         /* for multi devices, this assumes the ineffecient method that we allocate
866                          * all pixels on the device even though we only render to a subset */
867                         size_t offset = sizeof(uint8_t)*4*y*w;
868
869                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pmem.cuPBO);
870                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
871                         glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, (void*)offset);
872                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
873                         
874                         glEnable(GL_TEXTURE_2D);
875                         
876                         if(transparent) {
877                                 glEnable(GL_BLEND);
878                                 glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
879                         }
880
881                         glColor3f(1.0f, 1.0f, 1.0f);
882
883                         glPushMatrix();
884                         glTranslatef(0.0f, (float)dy, 0.0f);
885                                 
886                         glBegin(GL_QUADS);
887                         
888                         glTexCoord2f(0.0f, 0.0f);
889                         glVertex2f(0.0f, 0.0f);
890                         glTexCoord2f((float)w/(float)pmem.w, 0.0f);
891                         glVertex2f((float)width, 0.0f);
892                         glTexCoord2f((float)w/(float)pmem.w, (float)h/(float)pmem.h);
893                         glVertex2f((float)width, (float)height);
894                         glTexCoord2f(0.0f, (float)h/(float)pmem.h);
895                         glVertex2f(0.0f, (float)height);
896
897                         glEnd();
898
899                         glPopMatrix();
900
901                         if(transparent)
902                                 glDisable(GL_BLEND);
903                         
904                         glBindTexture(GL_TEXTURE_2D, 0);
905                         glDisable(GL_TEXTURE_2D);
906
907                         cuda_pop_context();
908
909                         return;
910                 }
911
912                 Device::draw_pixels(mem, y, w, h, dy, width, height, transparent);
913         }
914
915         void thread_run(DeviceTask *task)
916         {
917                 if(task->type == DeviceTask::PATH_TRACE) {
918                         RenderTile tile;
919                         
920                         bool progressive = task->integrator_progressive;
921                         
922                         /* keep rendering tiles until done */
923                         while(task->acquire_tile(this, tile)) {
924                                 int start_sample = tile.start_sample;
925                                 int end_sample = tile.start_sample + tile.num_samples;
926
927                                 for(int sample = start_sample; sample < end_sample; sample++) {
928                                         if (task->get_cancel()) {
929                                                 if(task->need_finish_queue == false)
930                                                         break;
931                                         }
932
933                                         path_trace(tile, sample, progressive);
934
935                                         tile.sample = sample + 1;
936
937                                         task->update_progress(tile);
938                                 }
939
940                                 task->release_tile(tile);
941                         }
942                 }
943                 else if(task->type == DeviceTask::SHADER) {
944                         shader(*task);
945
946                         cuda_push_context();
947                         cuda_assert(cuCtxSynchronize())
948                         cuda_pop_context();
949                 }
950         }
951
952         class CUDADeviceTask : public DeviceTask {
953         public:
954                 CUDADeviceTask(CUDADevice *device, DeviceTask& task)
955                 : DeviceTask(task)
956                 {
957                         run = function_bind(&CUDADevice::thread_run, device, this);
958                 }
959         };
960
961         void task_add(DeviceTask& task)
962         {
963                 if(task.type == DeviceTask::TONEMAP) {
964                         /* must be done in main thread due to opengl access */
965                         tonemap(task, task.buffer, task.rgba);
966
967                         cuda_push_context();
968                         cuda_assert(cuCtxSynchronize())
969                         cuda_pop_context();
970                 }
971                 else {
972                         task_pool.push(new CUDADeviceTask(this, task));
973                 }
974         }
975
976         void task_wait()
977         {
978                 task_pool.wait_work();
979         }
980
981         void task_cancel()
982         {
983                 task_pool.cancel();
984         }
985 };
986
987 Device *device_cuda_create(DeviceInfo& info, Stats &stats, bool background)
988 {
989         return new CUDADevice(info, stats, background);
990 }
991
992 void device_cuda_info(vector<DeviceInfo>& devices)
993 {
994         CUresult result;
995         int count = 0;
996
997         result = cuInit(0);
998         if(result != CUDA_SUCCESS) {
999                 if(result != CUDA_ERROR_NO_DEVICE)
1000                         fprintf(stderr, "CUDA cuInit: %s\n", CUDADevice::cuda_error_string(result));
1001                 return;
1002         }
1003
1004         result = cuDeviceGetCount(&count);
1005         if(result != CUDA_SUCCESS) {
1006                 fprintf(stderr, "CUDA cuDeviceGetCount: %s\n", CUDADevice::cuda_error_string(result));
1007                 return;
1008         }
1009         
1010         vector<DeviceInfo> display_devices;
1011         
1012         for(int num = 0; num < count; num++) {
1013                 char name[256];
1014                 int attr;
1015                 
1016                 if(cuDeviceGetName(name, 256, num) != CUDA_SUCCESS)
1017                         continue;
1018
1019                 DeviceInfo info;
1020
1021                 info.type = DEVICE_CUDA;
1022                 info.description = string(name);
1023                 info.id = string_printf("CUDA_%d", num);
1024                 info.num = num;
1025
1026                 int major, minor;
1027                 cuDeviceComputeCapability(&major, &minor, num);
1028                 info.advanced_shading = (major >= 2);
1029                 info.extended_images = (major >= 3);
1030                 info.pack_images = false;
1031
1032                 /* if device has a kernel timeout, assume it is used for display */
1033                 if(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, num) == CUDA_SUCCESS && attr == 1) {
1034                         info.display_device = true;
1035                         display_devices.push_back(info);
1036                 }
1037                 else
1038                         devices.push_back(info);
1039         }
1040
1041         if(!display_devices.empty())
1042                 devices.insert(devices.end(), display_devices.begin(), display_devices.end());
1043 }
1044
1045 CCL_NAMESPACE_END
1046