add BLI_strcpy_rlen, replace strcat, which was used in misleading way.
[blender.git] / intern / cycles / device / device_cuda.cpp
1 /*
2  * Copyright 2011, Blender Foundation.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  */
18
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22
23 #include "device.h"
24 #include "device_intern.h"
25
26 #include "buffers.h"
27
28 #include "util_cuda.h"
29 #include "util_debug.h"
30 #include "util_map.h"
31 #include "util_opengl.h"
32 #include "util_path.h"
33 #include "util_system.h"
34 #include "util_types.h"
35 #include "util_time.h"
36
37 CCL_NAMESPACE_BEGIN
38
39 class CUDADevice : public Device
40 {
41 public:
42         TaskPool task_pool;
43         CUdevice cuDevice;
44         CUcontext cuContext;
45         CUmodule cuModule;
46         map<device_ptr, bool> tex_interp_map;
47         int cuDevId;
48         bool first_error;
49
50         struct PixelMem {
51                 GLuint cuPBO;
52                 CUgraphicsResource cuPBOresource;
53                 GLuint cuTexId;
54                 int w, h;
55         };
56
57         map<device_ptr, PixelMem> pixel_mem_map;
58
59         CUdeviceptr cuda_device_ptr(device_ptr mem)
60         {
61                 return (CUdeviceptr)mem;
62         }
63
64         static const char *cuda_error_string(CUresult result)
65         {
66                 switch(result) {
67                         case CUDA_SUCCESS: return "No errors";
68                         case CUDA_ERROR_INVALID_VALUE: return "Invalid value";
69                         case CUDA_ERROR_OUT_OF_MEMORY: return "Out of memory";
70                         case CUDA_ERROR_NOT_INITIALIZED: return "Driver not initialized";
71                         case CUDA_ERROR_DEINITIALIZED: return "Driver deinitialized";
72
73                         case CUDA_ERROR_NO_DEVICE: return "No CUDA-capable device available";
74                         case CUDA_ERROR_INVALID_DEVICE: return "Invalid device";
75
76                         case CUDA_ERROR_INVALID_IMAGE: return "Invalid kernel image";
77                         case CUDA_ERROR_INVALID_CONTEXT: return "Invalid context";
78                         case CUDA_ERROR_CONTEXT_ALREADY_CURRENT: return "Context already current";
79                         case CUDA_ERROR_MAP_FAILED: return "Map failed";
80                         case CUDA_ERROR_UNMAP_FAILED: return "Unmap failed";
81                         case CUDA_ERROR_ARRAY_IS_MAPPED: return "Array is mapped";
82                         case CUDA_ERROR_ALREADY_MAPPED: return "Already mapped";
83                         case CUDA_ERROR_NO_BINARY_FOR_GPU: return "No binary for GPU";
84                         case CUDA_ERROR_ALREADY_ACQUIRED: return "Already acquired";
85                         case CUDA_ERROR_NOT_MAPPED: return "Not mapped";
86                         case CUDA_ERROR_NOT_MAPPED_AS_ARRAY: return "Mapped resource not available for access as an array";
87                         case CUDA_ERROR_NOT_MAPPED_AS_POINTER: return "Mapped resource not available for access as a pointer";
88                         case CUDA_ERROR_ECC_UNCORRECTABLE: return "Uncorrectable ECC error detected";
89                         case CUDA_ERROR_UNSUPPORTED_LIMIT: return "CUlimit not supported by device";
90
91                         case CUDA_ERROR_INVALID_SOURCE: return "Invalid source";
92                         case CUDA_ERROR_FILE_NOT_FOUND: return "File not found";
93                         case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND: return "Link to a shared object failed to resolve";
94                         case CUDA_ERROR_SHARED_OBJECT_INIT_FAILED: return "Shared object initialization failed";
95
96                         case CUDA_ERROR_INVALID_HANDLE: return "Invalid handle";
97
98                         case CUDA_ERROR_NOT_FOUND: return "Not found";
99
100                         case CUDA_ERROR_NOT_READY: return "CUDA not ready";
101
102                         case CUDA_ERROR_LAUNCH_FAILED: return "Launch failed";
103                         case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES: return "Launch exceeded resources";
104                         case CUDA_ERROR_LAUNCH_TIMEOUT: return "Launch exceeded timeout";
105                         case CUDA_ERROR_LAUNCH_INCOMPATIBLE_TEXTURING: return "Launch with incompatible texturing";
106
107                         case CUDA_ERROR_UNKNOWN: return "Unknown error";
108
109                         default: return "Unknown CUDA error value";
110                 }
111         }
112
113 /*#ifdef NDEBUG
114 #define cuda_abort()
115 #else
116 #define cuda_abort() abort()
117 #endif*/
118         void cuda_error_documentation()
119         {
120                 if(first_error) {
121                         fprintf(stderr, "\nRefer to the Cycles GPU rendering documentation for possible solutions:\n");
122                         fprintf(stderr, "http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/GPU_Rendering\n\n");
123                         first_error = false;
124                 }
125         }
126
127 #define cuda_assert(stmt) \
128         { \
129                 CUresult result = stmt; \
130                 \
131                 if(result != CUDA_SUCCESS) { \
132                         string message = string_printf("CUDA error: %s in %s", cuda_error_string(result), #stmt); \
133                         if(error_msg == "") \
134                                 error_msg = message; \
135                         fprintf(stderr, "%s\n", message.c_str()); \
136                         /*cuda_abort();*/ \
137                         cuda_error_documentation(); \
138                 } \
139         }
140
141         bool cuda_error_(CUresult result, const string& stmt)
142         {
143                 if(result == CUDA_SUCCESS)
144                         return false;
145
146                 string message = string_printf("CUDA error at %s: %s", stmt.c_str(), cuda_error_string(result));
147                 if(error_msg == "")
148                         error_msg = message;
149                 fprintf(stderr, "%s\n", message.c_str());
150                 cuda_error_documentation();
151                 return true;
152         }
153
154 #define cuda_error(stmt) cuda_error_(stmt, #stmt)
155
156         void cuda_error_message(const string& message)
157         {
158                 if(error_msg == "")
159                         error_msg = message;
160                 fprintf(stderr, "%s\n", message.c_str());
161                 cuda_error_documentation();
162         }
163
164         void cuda_push_context()
165         {
166                 cuda_assert(cuCtxSetCurrent(cuContext))
167         }
168
169         void cuda_pop_context()
170         {
171                 cuda_assert(cuCtxSetCurrent(NULL));
172         }
173
174         CUDADevice(DeviceInfo& info, Stats &stats, bool background_) : Device(stats)
175         {
176                 first_error = true;
177                 background = background_;
178
179                 cuDevId = info.num;
180                 cuDevice = 0;
181                 cuContext = 0;
182
183                 /* intialize */
184                 if(cuda_error(cuInit(0)))
185                         return;
186
187                 /* setup device and context */
188                 if(cuda_error(cuDeviceGet(&cuDevice, cuDevId)))
189                         return;
190
191                 CUresult result;
192
193                 if(background) {
194                         result = cuCtxCreate(&cuContext, 0, cuDevice);
195                 }
196                 else {
197                         result = cuGLCtxCreate(&cuContext, 0, cuDevice);
198
199                         if(result != CUDA_SUCCESS) {
200                                 result = cuCtxCreate(&cuContext, 0, cuDevice);
201                                 background = true;
202                         }
203                 }
204
205                 if(cuda_error_(result, "cuCtxCreate"))
206                         return;
207
208                 cuda_pop_context();
209         }
210
211         ~CUDADevice()
212         {
213                 task_pool.stop();
214
215                 cuda_push_context();
216                 cuda_assert(cuCtxDetach(cuContext))
217         }
218
219         bool support_device(bool experimental)
220         {
221                 if(!experimental) {
222                         int major, minor;
223                         cuDeviceComputeCapability(&major, &minor, cuDevId);
224
225                         if(major < 2) {
226                                 cuda_error_message(string_printf("CUDA device supported only with compute capability 2.0 or up, found %d.%d.", major, minor));
227                                 return false;
228                         }
229                 }
230
231                 return true;
232         }
233
234         string compile_kernel()
235         {
236                 /* compute cubin name */
237                 int major, minor;
238                 cuDeviceComputeCapability(&major, &minor, cuDevId);
239
240                 /* attempt to use kernel provided with blender */
241                 string cubin = path_get(string_printf("lib/kernel_sm_%d%d.cubin", major, minor));
242                 if(path_exists(cubin))
243                         return cubin;
244
245                 /* not found, try to use locally compiled kernel */
246                 string kernel_path = path_get("kernel");
247                 string md5 = path_files_md5_hash(kernel_path);
248
249                 cubin = string_printf("cycles_kernel_sm%d%d_%s.cubin", major, minor, md5.c_str());
250                 cubin = path_user_get(path_join("cache", cubin));
251
252                 /* if exists already, use it */
253                 if(path_exists(cubin))
254                         return cubin;
255
256 #ifdef _WIN32
257                 if(cuHavePrecompiledKernels()) {
258                         if(major < 2)
259                                 cuda_error_message(string_printf("CUDA device requires compute capability 2.0 or up, found %d.%d. Your GPU is not supported.", major, minor));
260                         else
261                                 cuda_error_message(string_printf("CUDA binary kernel for this graphics card compute capability (%d.%d) not found.", major, minor));
262                         return "";
263                 }
264 #endif
265
266                 /* if not, find CUDA compiler */
267                 string nvcc = cuCompilerPath();
268
269                 if(nvcc == "") {
270                         cuda_error_message("CUDA nvcc compiler not found. Install CUDA toolkit in default location.");
271                         return "";
272                 }
273
274                 /* compile */
275                 string kernel = path_join(kernel_path, "kernel.cu");
276                 string include = kernel_path;
277                 const int machine = system_cpu_bits();
278                 const int maxreg = 24;
279
280                 double starttime = time_dt();
281                 printf("Compiling CUDA kernel ...\n");
282
283                 path_create_directories(cubin);
284
285                 string command = string_printf("\"%s\" -arch=sm_%d%d -m%d --cubin \"%s\" "
286                         "-o \"%s\" --ptxas-options=\"-v\" --maxrregcount=%d --opencc-options -OPT:Olimit=0 -I\"%s\" -DNVCC",
287                         nvcc.c_str(), major, minor, machine, kernel.c_str(), cubin.c_str(), maxreg, include.c_str());
288
289                 if(system(command.c_str()) == -1) {
290                         cuda_error_message("Failed to execute compilation command, see console for details.");
291                         return "";
292                 }
293
294                 /* verify if compilation succeeded */
295                 if(!path_exists(cubin)) {
296                         cuda_error_message("CUDA kernel compilation failed, see console for details.");
297                         return "";
298                 }
299
300                 printf("Kernel compilation finished in %.2lfs.\n", time_dt() - starttime);
301
302                 return cubin;
303         }
304
305         bool load_kernels(bool experimental)
306         {
307                 /* check if cuda init succeeded */
308                 if(cuContext == 0)
309                         return false;
310                 
311                 /* check if GPU is supported with current feature set */
312                 if(!support_device(experimental))
313                         return false;
314
315                 /* get kernel */
316                 string cubin = compile_kernel();
317
318                 if(cubin == "")
319                         return false;
320
321                 /* open module */
322                 cuda_push_context();
323
324                 CUresult result = cuModuleLoad(&cuModule, cubin.c_str());
325                 if(cuda_error_(result, "cuModuleLoad"))
326                         cuda_error_message(string_printf("Failed loading CUDA kernel %s.", cubin.c_str()));
327
328                 cuda_pop_context();
329
330                 return (result == CUDA_SUCCESS);
331         }
332
333         void mem_alloc(device_memory& mem, MemoryType type)
334         {
335                 cuda_push_context();
336                 CUdeviceptr device_pointer;
337                 size_t size = mem.memory_size();
338                 cuda_assert(cuMemAlloc(&device_pointer, size))
339                 mem.device_pointer = (device_ptr)device_pointer;
340                 stats.mem_alloc(size);
341                 cuda_pop_context();
342         }
343
344         void mem_copy_to(device_memory& mem)
345         {
346                 cuda_push_context();
347                 if(mem.device_pointer)
348                         cuda_assert(cuMemcpyHtoD(cuda_device_ptr(mem.device_pointer), (void*)mem.data_pointer, mem.memory_size()))
349                 cuda_pop_context();
350         }
351
352         void mem_copy_from(device_memory& mem, int y, int w, int h, int elem)
353         {
354                 size_t offset = elem*y*w;
355                 size_t size = elem*w*h;
356
357                 cuda_push_context();
358                 if(mem.device_pointer) {
359                         cuda_assert(cuMemcpyDtoH((uchar*)mem.data_pointer + offset,
360                                 (CUdeviceptr)((uchar*)mem.device_pointer + offset), size))
361                 }
362                 else {
363                         memset((char*)mem.data_pointer + offset, 0, size);
364                 }
365                 cuda_pop_context();
366         }
367
368         void mem_zero(device_memory& mem)
369         {
370                 memset((void*)mem.data_pointer, 0, mem.memory_size());
371
372                 cuda_push_context();
373                 if(mem.device_pointer)
374                         cuda_assert(cuMemsetD8(cuda_device_ptr(mem.device_pointer), 0, mem.memory_size()))
375                 cuda_pop_context();
376         }
377
378         void mem_free(device_memory& mem)
379         {
380                 if(mem.device_pointer) {
381                         cuda_push_context();
382                         cuda_assert(cuMemFree(cuda_device_ptr(mem.device_pointer)))
383                         cuda_pop_context();
384
385                         mem.device_pointer = 0;
386
387                         stats.mem_free(mem.memory_size());
388                 }
389         }
390
391         void const_copy_to(const char *name, void *host, size_t size)
392         {
393                 CUdeviceptr mem;
394                 size_t bytes;
395
396                 cuda_push_context();
397                 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule, name))
398                 //assert(bytes == size);
399                 cuda_assert(cuMemcpyHtoD(mem, host, size))
400                 cuda_pop_context();
401         }
402
403         void tex_alloc(const char *name, device_memory& mem, bool interpolation, bool periodic)
404         {
405                 /* determine format */
406                 CUarray_format_enum format;
407                 size_t dsize = datatype_size(mem.data_type);
408                 size_t size = mem.memory_size();
409
410                 switch(mem.data_type) {
411                         case TYPE_UCHAR: format = CU_AD_FORMAT_UNSIGNED_INT8; break;
412                         case TYPE_UINT: format = CU_AD_FORMAT_UNSIGNED_INT32; break;
413                         case TYPE_INT: format = CU_AD_FORMAT_SIGNED_INT32; break;
414                         case TYPE_FLOAT: format = CU_AD_FORMAT_FLOAT; break;
415                         default: assert(0); return;
416                 }
417
418                 CUtexref texref = NULL;
419
420                 cuda_push_context();
421                 cuda_assert(cuModuleGetTexRef(&texref, cuModule, name))
422
423                 if(!texref) {
424                         cuda_pop_context();
425                         return;
426                 }
427
428                 if(interpolation) {
429                         CUarray handle = NULL;
430                         CUDA_ARRAY_DESCRIPTOR desc;
431
432                         desc.Width = mem.data_width;
433                         desc.Height = mem.data_height;
434                         desc.Format = format;
435                         desc.NumChannels = mem.data_elements;
436
437                         cuda_assert(cuArrayCreate(&handle, &desc))
438
439                         if(!handle) {
440                                 cuda_pop_context();
441                                 return;
442                         }
443
444                         if(mem.data_height > 1) {
445                                 CUDA_MEMCPY2D param;
446                                 memset(&param, 0, sizeof(param));
447                                 param.dstMemoryType = CU_MEMORYTYPE_ARRAY;
448                                 param.dstArray = handle;
449                                 param.srcMemoryType = CU_MEMORYTYPE_HOST;
450                                 param.srcHost = (void*)mem.data_pointer;
451                                 param.srcPitch = mem.data_width*dsize*mem.data_elements;
452                                 param.WidthInBytes = param.srcPitch;
453                                 param.Height = mem.data_height;
454
455                                 cuda_assert(cuMemcpy2D(&param))
456                         }
457                         else
458                                 cuda_assert(cuMemcpyHtoA(handle, 0, (void*)mem.data_pointer, size))
459
460                         cuda_assert(cuTexRefSetArray(texref, handle, CU_TRSA_OVERRIDE_FORMAT))
461
462                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_LINEAR))
463                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_NORMALIZED_COORDINATES))
464
465                         mem.device_pointer = (device_ptr)handle;
466
467                         stats.mem_alloc(size);
468                 }
469                 else {
470                         cuda_pop_context();
471
472                         mem_alloc(mem, MEM_READ_ONLY);
473                         mem_copy_to(mem);
474
475                         cuda_push_context();
476
477                         cuda_assert(cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size))
478                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_POINT))
479                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_READ_AS_INTEGER))
480                 }
481
482                 if(periodic) {
483                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_WRAP))
484                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_WRAP))
485                 }
486                 else {
487                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_CLAMP))
488                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_CLAMP))
489                 }
490                 cuda_assert(cuTexRefSetFormat(texref, format, mem.data_elements))
491
492                 cuda_pop_context();
493
494                 tex_interp_map[mem.device_pointer] = interpolation;
495         }
496
497         void tex_free(device_memory& mem)
498         {
499                 if(mem.device_pointer) {
500                         if(tex_interp_map[mem.device_pointer]) {
501                                 cuda_push_context();
502                                 cuArrayDestroy((CUarray)mem.device_pointer);
503                                 cuda_pop_context();
504
505                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
506                                 mem.device_pointer = 0;
507
508                                 stats.mem_free(mem.memory_size());
509                         }
510                         else {
511                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
512                                 mem_free(mem);
513                         }
514                 }
515         }
516
517         void path_trace(RenderTile& rtile, int sample)
518         {
519                 if(have_error())
520                         return;
521
522                 cuda_push_context();
523
524                 CUfunction cuPathTrace;
525                 CUdeviceptr d_buffer = cuda_device_ptr(rtile.buffer);
526                 CUdeviceptr d_rng_state = cuda_device_ptr(rtile.rng_state);
527
528                 /* get kernel function */
529                 cuda_assert(cuModuleGetFunction(&cuPathTrace, cuModule, "kernel_cuda_path_trace"))
530                 
531                 /* pass in parameters */
532                 int offset = 0;
533                 
534                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_buffer, sizeof(d_buffer)))
535                 offset += sizeof(d_buffer);
536
537                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_rng_state, sizeof(d_rng_state)))
538                 offset += sizeof(d_rng_state);
539
540                 offset = align_up(offset, __alignof(sample));
541
542                 cuda_assert(cuParamSeti(cuPathTrace, offset, sample))
543                 offset += sizeof(sample);
544
545                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.x))
546                 offset += sizeof(rtile.x);
547
548                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.y))
549                 offset += sizeof(rtile.y);
550
551                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.w))
552                 offset += sizeof(rtile.w);
553
554                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.h))
555                 offset += sizeof(rtile.h);
556
557                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.offset))
558                 offset += sizeof(rtile.offset);
559
560                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.stride))
561                 offset += sizeof(rtile.stride);
562
563                 cuda_assert(cuParamSetSize(cuPathTrace, offset))
564
565                 /* launch kernel: todo find optimal size, cache config for fermi */
566 #ifndef __APPLE__
567                 int xthreads = 16;
568                 int ythreads = 16;
569 #else
570                 int xthreads = 8;
571                 int ythreads = 8;
572 #endif
573                 int xblocks = (rtile.w + xthreads - 1)/xthreads;
574                 int yblocks = (rtile.h + ythreads - 1)/ythreads;
575
576                 cuda_assert(cuFuncSetCacheConfig(cuPathTrace, CU_FUNC_CACHE_PREFER_L1))
577                 cuda_assert(cuFuncSetBlockShape(cuPathTrace, xthreads, ythreads, 1))
578                 cuda_assert(cuLaunchGrid(cuPathTrace, xblocks, yblocks))
579
580                 cuda_assert(cuCtxSynchronize())
581
582                 cuda_pop_context();
583         }
584
585         void tonemap(DeviceTask& task, device_ptr buffer, device_ptr rgba)
586         {
587                 if(have_error())
588                         return;
589
590                 cuda_push_context();
591
592                 CUfunction cuFilmConvert;
593                 CUdeviceptr d_rgba = map_pixels(rgba);
594                 CUdeviceptr d_buffer = cuda_device_ptr(buffer);
595
596                 /* get kernel function */
597                 cuda_assert(cuModuleGetFunction(&cuFilmConvert, cuModule, "kernel_cuda_tonemap"))
598
599                 /* pass in parameters */
600                 int offset = 0;
601
602                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_rgba, sizeof(d_rgba)))
603                 offset += sizeof(d_rgba);
604                 
605                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_buffer, sizeof(d_buffer)))
606                 offset += sizeof(d_buffer);
607
608                 int sample = task.sample;
609                 offset = align_up(offset, __alignof(sample));
610
611                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.sample))
612                 offset += sizeof(task.sample);
613
614                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.x))
615                 offset += sizeof(task.x);
616
617                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.y))
618                 offset += sizeof(task.y);
619
620                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.w))
621                 offset += sizeof(task.w);
622
623                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.h))
624                 offset += sizeof(task.h);
625
626                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.offset))
627                 offset += sizeof(task.offset);
628
629                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.stride))
630                 offset += sizeof(task.stride);
631
632                 cuda_assert(cuParamSetSize(cuFilmConvert, offset))
633
634                 /* launch kernel: todo find optimal size, cache config for fermi */
635 #ifndef __APPLE__
636                 int xthreads = 16;
637                 int ythreads = 16;
638 #else
639                 int xthreads = 8;
640                 int ythreads = 8;
641 #endif
642                 int xblocks = (task.w + xthreads - 1)/xthreads;
643                 int yblocks = (task.h + ythreads - 1)/ythreads;
644
645                 cuda_assert(cuFuncSetCacheConfig(cuFilmConvert, CU_FUNC_CACHE_PREFER_L1))
646                 cuda_assert(cuFuncSetBlockShape(cuFilmConvert, xthreads, ythreads, 1))
647                 cuda_assert(cuLaunchGrid(cuFilmConvert, xblocks, yblocks))
648
649                 unmap_pixels(task.rgba);
650
651                 cuda_pop_context();
652         }
653
654         void shader(DeviceTask& task)
655         {
656                 if(have_error())
657                         return;
658
659                 cuda_push_context();
660
661                 CUfunction cuDisplace;
662                 CUdeviceptr d_input = cuda_device_ptr(task.shader_input);
663                 CUdeviceptr d_offset = cuda_device_ptr(task.shader_output);
664
665                 /* get kernel function */
666                 cuda_assert(cuModuleGetFunction(&cuDisplace, cuModule, "kernel_cuda_shader"))
667                 
668                 /* pass in parameters */
669                 int offset = 0;
670                 
671                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_input, sizeof(d_input)))
672                 offset += sizeof(d_input);
673
674                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_offset, sizeof(d_offset)))
675                 offset += sizeof(d_offset);
676
677                 int shader_eval_type = task.shader_eval_type;
678                 offset = align_up(offset, __alignof(shader_eval_type));
679
680                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_eval_type))
681                 offset += sizeof(task.shader_eval_type);
682
683                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_x))
684                 offset += sizeof(task.shader_x);
685
686                 cuda_assert(cuParamSetSize(cuDisplace, offset))
687
688                 /* launch kernel: todo find optimal size, cache config for fermi */
689 #ifndef __APPLE__
690                 int xthreads = 16;
691 #else
692                 int xthreads = 8;
693 #endif
694                 int xblocks = (task.shader_w + xthreads - 1)/xthreads;
695
696                 cuda_assert(cuFuncSetCacheConfig(cuDisplace, CU_FUNC_CACHE_PREFER_L1))
697                 cuda_assert(cuFuncSetBlockShape(cuDisplace, xthreads, 1, 1))
698                 cuda_assert(cuLaunchGrid(cuDisplace, xblocks, 1))
699
700                 cuda_pop_context();
701         }
702
703         CUdeviceptr map_pixels(device_ptr mem)
704         {
705                 if(!background) {
706                         PixelMem pmem = pixel_mem_map[mem];
707                         CUdeviceptr buffer;
708                         
709                         size_t bytes;
710                         cuda_assert(cuGraphicsMapResources(1, &pmem.cuPBOresource, 0))
711                         cuda_assert(cuGraphicsResourceGetMappedPointer(&buffer, &bytes, pmem.cuPBOresource))
712                         
713                         return buffer;
714                 }
715
716                 return cuda_device_ptr(mem);
717         }
718
719         void unmap_pixels(device_ptr mem)
720         {
721                 if(!background) {
722                         PixelMem pmem = pixel_mem_map[mem];
723
724                         cuda_assert(cuGraphicsUnmapResources(1, &pmem.cuPBOresource, 0))
725                 }
726         }
727
728         void pixels_alloc(device_memory& mem)
729         {
730                 if(!background) {
731                         PixelMem pmem;
732
733                         pmem.w = mem.data_width;
734                         pmem.h = mem.data_height;
735
736                         cuda_push_context();
737
738                         glGenBuffers(1, &pmem.cuPBO);
739                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
740                         glBufferData(GL_PIXEL_UNPACK_BUFFER, pmem.w*pmem.h*sizeof(GLfloat)*3, NULL, GL_DYNAMIC_DRAW);
741                         
742                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
743                         
744                         glGenTextures(1, &pmem.cuTexId);
745                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
746                         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, pmem.w, pmem.h, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
747                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
748                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
749                         glBindTexture(GL_TEXTURE_2D, 0);
750                         
751                         CUresult result = cuGraphicsGLRegisterBuffer(&pmem.cuPBOresource, pmem.cuPBO, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
752
753                         if(result == CUDA_SUCCESS) {
754                                 cuda_pop_context();
755
756                                 mem.device_pointer = pmem.cuTexId;
757                                 pixel_mem_map[mem.device_pointer] = pmem;
758
759                                 stats.mem_alloc(mem.memory_size());
760
761                                 return;
762                         }
763                         else {
764                                 /* failed to register buffer, fallback to no interop */
765                                 glDeleteBuffers(1, &pmem.cuPBO);
766                                 glDeleteTextures(1, &pmem.cuTexId);
767
768                                 cuda_pop_context();
769
770                                 background = true;
771                         }
772                 }
773
774                 Device::pixels_alloc(mem);
775         }
776
777         void pixels_copy_from(device_memory& mem, int y, int w, int h)
778         {
779                 if(!background) {
780                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
781
782                         cuda_push_context();
783
784                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
785                         uchar *pixels = (uchar*)glMapBuffer(GL_PIXEL_UNPACK_BUFFER, GL_READ_ONLY);
786                         size_t offset = sizeof(uchar)*4*y*w;
787                         memcpy((uchar*)mem.data_pointer + offset, pixels + offset, sizeof(uchar)*4*w*h);
788                         glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
789                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
790
791                         cuda_pop_context();
792
793                         return;
794                 }
795
796                 Device::pixels_copy_from(mem, y, w, h);
797         }
798
799         void pixels_free(device_memory& mem)
800         {
801                 if(mem.device_pointer) {
802                         if(!background) {
803                                 PixelMem pmem = pixel_mem_map[mem.device_pointer];
804
805                                 cuda_push_context();
806
807                                 cuda_assert(cuGraphicsUnregisterResource(pmem.cuPBOresource))
808                                 glDeleteBuffers(1, &pmem.cuPBO);
809                                 glDeleteTextures(1, &pmem.cuTexId);
810
811                                 cuda_pop_context();
812
813                                 pixel_mem_map.erase(pixel_mem_map.find(mem.device_pointer));
814                                 mem.device_pointer = 0;
815
816                                 stats.mem_free(mem.memory_size());
817
818                                 return;
819                         }
820
821                         Device::pixels_free(mem);
822                 }
823         }
824
825         void draw_pixels(device_memory& mem, int y, int w, int h, int dy, int width, int height, bool transparent)
826         {
827                 if(!background) {
828                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
829
830                         cuda_push_context();
831
832                         /* for multi devices, this assumes the ineffecient method that we allocate
833                          * all pixels on the device even though we only render to a subset */
834                         size_t offset = sizeof(uint8_t)*4*y*w;
835
836                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pmem.cuPBO);
837                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
838                         glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, (void*)offset);
839                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
840                         
841                         glEnable(GL_TEXTURE_2D);
842                         
843                         if(transparent) {
844                                 glEnable(GL_BLEND);
845                                 glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
846                         }
847
848                         glColor3f(1.0f, 1.0f, 1.0f);
849
850                         glPushMatrix();
851                         glTranslatef(0.0f, (float)dy, 0.0f);
852                                 
853                         glBegin(GL_QUADS);
854                         
855                         glTexCoord2f(0.0f, 0.0f);
856                         glVertex2f(0.0f, 0.0f);
857                         glTexCoord2f((float)w/(float)pmem.w, 0.0f);
858                         glVertex2f((float)width, 0.0f);
859                         glTexCoord2f((float)w/(float)pmem.w, (float)h/(float)pmem.h);
860                         glVertex2f((float)width, (float)height);
861                         glTexCoord2f(0.0f, (float)h/(float)pmem.h);
862                         glVertex2f(0.0f, (float)height);
863
864                         glEnd();
865
866                         glPopMatrix();
867
868                         if(transparent)
869                                 glDisable(GL_BLEND);
870                         
871                         glBindTexture(GL_TEXTURE_2D, 0);
872                         glDisable(GL_TEXTURE_2D);
873
874                         cuda_pop_context();
875
876                         return;
877                 }
878
879                 Device::draw_pixels(mem, y, w, h, dy, width, height, transparent);
880         }
881
882         void thread_run(DeviceTask *task)
883         {
884                 if(task->type == DeviceTask::PATH_TRACE) {
885                         RenderTile tile;
886                         
887                         /* keep rendering tiles until done */
888                         while(task->acquire_tile(this, tile)) {
889                                 int start_sample = tile.start_sample;
890                                 int end_sample = tile.start_sample + tile.num_samples;
891
892                                 for(int sample = start_sample; sample < end_sample; sample++) {
893                                         if (task->get_cancel()) {
894                                                 if(task->need_finish_queue == false)
895                                                         break;
896                                         }
897
898                                         path_trace(tile, sample);
899
900                                         tile.sample = sample + 1;
901
902                                         task->update_progress(tile);
903                                 }
904
905                                 task->release_tile(tile);
906                         }
907                 }
908                 else if(task->type == DeviceTask::SHADER) {
909                         shader(*task);
910
911                         cuda_push_context();
912                         cuda_assert(cuCtxSynchronize())
913                         cuda_pop_context();
914                 }
915         }
916
917         class CUDADeviceTask : public DeviceTask {
918         public:
919                 CUDADeviceTask(CUDADevice *device, DeviceTask& task)
920                 : DeviceTask(task)
921                 {
922                         run = function_bind(&CUDADevice::thread_run, device, this);
923                 }
924         };
925
926         void task_add(DeviceTask& task)
927         {
928                 if(task.type == DeviceTask::TONEMAP) {
929                         /* must be done in main thread due to opengl access */
930                         tonemap(task, task.buffer, task.rgba);
931
932                         cuda_push_context();
933                         cuda_assert(cuCtxSynchronize())
934                         cuda_pop_context();
935                 }
936                 else {
937                         task_pool.push(new CUDADeviceTask(this, task));
938                 }
939         }
940
941         void task_wait()
942         {
943                 task_pool.wait_work();
944         }
945
946         void task_cancel()
947         {
948                 task_pool.cancel();
949         }
950 };
951
952 Device *device_cuda_create(DeviceInfo& info, Stats &stats, bool background)
953 {
954         return new CUDADevice(info, stats, background);
955 }
956
957 void device_cuda_info(vector<DeviceInfo>& devices)
958 {
959         CUresult result;
960         int count = 0;
961
962         result = cuInit(0);
963         if(result != CUDA_SUCCESS) {
964                 if(result != CUDA_ERROR_NO_DEVICE)
965                         fprintf(stderr, "CUDA cuInit: %s\n", CUDADevice::cuda_error_string(result));
966                 return;
967         }
968
969         result = cuDeviceGetCount(&count);
970         if(result != CUDA_SUCCESS) {
971                 fprintf(stderr, "CUDA cuDeviceGetCount: %s\n", CUDADevice::cuda_error_string(result));
972                 return;
973         }
974         
975         vector<DeviceInfo> display_devices;
976         
977         for(int num = 0; num < count; num++) {
978                 char name[256];
979                 int attr;
980                 
981                 if(cuDeviceGetName(name, 256, num) != CUDA_SUCCESS)
982                         continue;
983
984                 DeviceInfo info;
985
986                 info.type = DEVICE_CUDA;
987                 info.description = string(name);
988                 info.id = string_printf("CUDA_%d", num);
989                 info.num = num;
990
991                 int major, minor;
992                 cuDeviceComputeCapability(&major, &minor, num);
993                 info.advanced_shading = (major >= 2);
994                 info.pack_images = false;
995
996                 /* if device has a kernel timeout, assume it is used for display */
997                 if(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, num) == CUDA_SUCCESS && attr == 1) {
998                         info.display_device = true;
999                         display_devices.push_back(info);
1000                 }
1001                 else
1002                         devices.push_back(info);
1003         }
1004
1005         if(!display_devices.empty())
1006                 devices.insert(devices.end(), display_devices.begin(), display_devices.end());
1007 }
1008
1009 CCL_NAMESPACE_END
1010