Merged revision(s) 57768-57827 from trunk/blender into soc-2013-dingto
[blender-staging.git] / intern / cycles / device / device_cuda.cpp
1 /*
2  * Copyright 2011, Blender Foundation.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  */
18
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22
23 #include "device.h"
24 #include "device_intern.h"
25
26 #include "buffers.h"
27
28 #include "util_cuda.h"
29 #include "util_debug.h"
30 #include "util_map.h"
31 #include "util_opengl.h"
32 #include "util_path.h"
33 #include "util_system.h"
34 #include "util_types.h"
35 #include "util_time.h"
36
37 CCL_NAMESPACE_BEGIN
38
39 class CUDADevice : public Device
40 {
41 public:
42         TaskPool task_pool;
43         CUdevice cuDevice;
44         CUcontext cuContext;
45         CUmodule cuModule;
46         map<device_ptr, bool> tex_interp_map;
47         int cuDevId;
48         bool first_error;
49
50         struct PixelMem {
51                 GLuint cuPBO;
52                 CUgraphicsResource cuPBOresource;
53                 GLuint cuTexId;
54                 int w, h;
55         };
56
57         map<device_ptr, PixelMem> pixel_mem_map;
58
59         CUdeviceptr cuda_device_ptr(device_ptr mem)
60         {
61                 return (CUdeviceptr)mem;
62         }
63
64         static const char *cuda_error_string(CUresult result)
65         {
66                 switch(result) {
67                         case CUDA_SUCCESS: return "No errors";
68                         case CUDA_ERROR_INVALID_VALUE: return "Invalid value";
69                         case CUDA_ERROR_OUT_OF_MEMORY: return "Out of memory";
70                         case CUDA_ERROR_NOT_INITIALIZED: return "Driver not initialized";
71                         case CUDA_ERROR_DEINITIALIZED: return "Driver deinitialized";
72
73                         case CUDA_ERROR_NO_DEVICE: return "No CUDA-capable device available";
74                         case CUDA_ERROR_INVALID_DEVICE: return "Invalid device";
75
76                         case CUDA_ERROR_INVALID_IMAGE: return "Invalid kernel image";
77                         case CUDA_ERROR_INVALID_CONTEXT: return "Invalid context";
78                         case CUDA_ERROR_CONTEXT_ALREADY_CURRENT: return "Context already current";
79                         case CUDA_ERROR_MAP_FAILED: return "Map failed";
80                         case CUDA_ERROR_UNMAP_FAILED: return "Unmap failed";
81                         case CUDA_ERROR_ARRAY_IS_MAPPED: return "Array is mapped";
82                         case CUDA_ERROR_ALREADY_MAPPED: return "Already mapped";
83                         case CUDA_ERROR_NO_BINARY_FOR_GPU: return "No binary for GPU";
84                         case CUDA_ERROR_ALREADY_ACQUIRED: return "Already acquired";
85                         case CUDA_ERROR_NOT_MAPPED: return "Not mapped";
86                         case CUDA_ERROR_NOT_MAPPED_AS_ARRAY: return "Mapped resource not available for access as an array";
87                         case CUDA_ERROR_NOT_MAPPED_AS_POINTER: return "Mapped resource not available for access as a pointer";
88                         case CUDA_ERROR_ECC_UNCORRECTABLE: return "Uncorrectable ECC error detected";
89                         case CUDA_ERROR_UNSUPPORTED_LIMIT: return "CUlimit not supported by device";
90
91                         case CUDA_ERROR_INVALID_SOURCE: return "Invalid source";
92                         case CUDA_ERROR_FILE_NOT_FOUND: return "File not found";
93                         case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND: return "Link to a shared object failed to resolve";
94                         case CUDA_ERROR_SHARED_OBJECT_INIT_FAILED: return "Shared object initialization failed";
95
96                         case CUDA_ERROR_INVALID_HANDLE: return "Invalid handle";
97
98                         case CUDA_ERROR_NOT_FOUND: return "Not found";
99
100                         case CUDA_ERROR_NOT_READY: return "CUDA not ready";
101
102                         case CUDA_ERROR_LAUNCH_FAILED: return "Launch failed";
103                         case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES: return "Launch exceeded resources";
104                         case CUDA_ERROR_LAUNCH_TIMEOUT: return "Launch exceeded timeout";
105                         case CUDA_ERROR_LAUNCH_INCOMPATIBLE_TEXTURING: return "Launch with incompatible texturing";
106
107                         case CUDA_ERROR_UNKNOWN: return "Unknown error";
108
109                         default: return "Unknown CUDA error value";
110                 }
111         }
112
113 /*#ifdef NDEBUG
114 #define cuda_abort()
115 #else
116 #define cuda_abort() abort()
117 #endif*/
118         void cuda_error_documentation()
119         {
120                 if(first_error) {
121                         fprintf(stderr, "\nRefer to the Cycles GPU rendering documentation for possible solutions:\n");
122                         fprintf(stderr, "http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/GPU_Rendering\n\n");
123                         first_error = false;
124                 }
125         }
126
127 #define cuda_assert(stmt) \
128         { \
129                 CUresult result = stmt; \
130                 \
131                 if(result != CUDA_SUCCESS) { \
132                         string message = string_printf("CUDA error: %s in %s", cuda_error_string(result), #stmt); \
133                         if(error_msg == "") \
134                                 error_msg = message; \
135                         fprintf(stderr, "%s\n", message.c_str()); \
136                         /*cuda_abort();*/ \
137                         cuda_error_documentation(); \
138                 } \
139         }
140
141         bool cuda_error_(CUresult result, const string& stmt)
142         {
143                 if(result == CUDA_SUCCESS)
144                         return false;
145
146                 string message = string_printf("CUDA error at %s: %s", stmt.c_str(), cuda_error_string(result));
147                 if(error_msg == "")
148                         error_msg = message;
149                 fprintf(stderr, "%s\n", message.c_str());
150                 cuda_error_documentation();
151                 return true;
152         }
153
154 #define cuda_error(stmt) cuda_error_(stmt, #stmt)
155
156         void cuda_error_message(const string& message)
157         {
158                 if(error_msg == "")
159                         error_msg = message;
160                 fprintf(stderr, "%s\n", message.c_str());
161                 cuda_error_documentation();
162         }
163
164         void cuda_push_context()
165         {
166                 cuda_assert(cuCtxSetCurrent(cuContext))
167         }
168
169         void cuda_pop_context()
170         {
171                 cuda_assert(cuCtxSetCurrent(NULL));
172         }
173
174         CUDADevice(DeviceInfo& info, Stats &stats, bool background_) : Device(stats)
175         {
176                 first_error = true;
177                 background = background_;
178
179                 cuDevId = info.num;
180                 cuDevice = 0;
181                 cuContext = 0;
182
183                 /* intialize */
184                 if(cuda_error(cuInit(0)))
185                         return;
186
187                 /* setup device and context */
188                 if(cuda_error(cuDeviceGet(&cuDevice, cuDevId)))
189                         return;
190
191                 CUresult result;
192
193                 if(background) {
194                         result = cuCtxCreate(&cuContext, 0, cuDevice);
195                 }
196                 else {
197                         result = cuGLCtxCreate(&cuContext, 0, cuDevice);
198
199                         if(result != CUDA_SUCCESS) {
200                                 result = cuCtxCreate(&cuContext, 0, cuDevice);
201                                 background = true;
202                         }
203                 }
204
205                 if(cuda_error_(result, "cuCtxCreate"))
206                         return;
207
208                 cuda_pop_context();
209         }
210
211         ~CUDADevice()
212         {
213                 task_pool.stop();
214
215                 cuda_push_context();
216                 cuda_assert(cuCtxDetach(cuContext))
217         }
218
219         bool support_device(bool experimental)
220         {
221                 if(!experimental) {
222                         int major, minor;
223                         cuDeviceComputeCapability(&major, &minor, cuDevId);
224
225                         if(major < 2) {
226                                 cuda_error_message(string_printf("CUDA device supported only with compute capability 2.0 or up, found %d.%d.", major, minor));
227                                 return false;
228                         }
229                 }
230
231                 return true;
232         }
233
234         string compile_kernel()
235         {
236                 /* compute cubin name */
237                 int major, minor;
238                 cuDeviceComputeCapability(&major, &minor, cuDevId);
239
240                 /* attempt to use kernel provided with blender */
241                 string cubin = path_get(string_printf("lib/kernel_sm_%d%d.cubin", major, minor));
242                 if(path_exists(cubin))
243                         return cubin;
244
245                 /* not found, try to use locally compiled kernel */
246                 string kernel_path = path_get("kernel");
247                 string md5 = path_files_md5_hash(kernel_path);
248
249                 cubin = string_printf("cycles_kernel_sm%d%d_%s.cubin", major, minor, md5.c_str());
250                 cubin = path_user_get(path_join("cache", cubin));
251
252                 /* if exists already, use it */
253                 if(path_exists(cubin))
254                         return cubin;
255
256 #ifdef _WIN32
257                 if(cuHavePrecompiledKernels()) {
258                         if(major < 2)
259                                 cuda_error_message(string_printf("CUDA device requires compute capability 2.0 or up, found %d.%d. Your GPU is not supported.", major, minor));
260                         else
261                                 cuda_error_message(string_printf("CUDA binary kernel for this graphics card compute capability (%d.%d) not found.", major, minor));
262                         return "";
263                 }
264 #endif
265
266                 /* if not, find CUDA compiler */
267                 string nvcc = cuCompilerPath();
268
269                 if(nvcc == "") {
270                         cuda_error_message("CUDA nvcc compiler not found. Install CUDA toolkit in default location.");
271                         return "";
272                 }
273
274                 int cuda_version = cuCompilerVersion();
275
276                 if(cuda_version == 0) {
277                         cuda_error_message("CUDA nvcc compiler version could not be parsed.");
278                         return "";
279                 }
280
281                 if(cuda_version != 50)
282                         printf("CUDA version %d.%d detected, build may succeed but only CUDA 5.0 is officially supported.\n", cuda_version/10, cuda_version%10);
283
284                 /* compile */
285                 string kernel = path_join(kernel_path, "kernel.cu");
286                 string include = kernel_path;
287                 const int machine = system_cpu_bits();
288                 string arch_flags;
289
290                 /* build flags depending on CUDA version and arch */
291                 if(cuda_version < 50) {
292                         /* CUDA 4.x */
293                         if(major == 1) {
294                                 /* sm_1x */
295                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0";
296                         }
297                         else if(major == 2) {
298                                 /* sm_2x */
299                                 arch_flags = "--maxrregcount=24";
300                         }
301                         else {
302                                 /* sm_3x */
303                                 arch_flags = "--maxrregcount=32";
304                         }
305                 }
306                 else {
307                         /* CUDA 5.x */
308                         if(major == 1) {
309                                 /* sm_1x */
310                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0 --use_fast_math";
311                         }
312                         else if(major == 2) {
313                                 /* sm_2x */
314                                 arch_flags = "--maxrregcount=32 --use_fast_math";
315                         }
316                         else {
317                                 /* sm_3x */
318                                 arch_flags = "--maxrregcount=32 --use_fast_math";
319                         }
320                 }
321
322                 double starttime = time_dt();
323                 printf("Compiling CUDA kernel ...\n");
324
325                 path_create_directories(cubin);
326
327                 string command = string_printf("\"%s\" -arch=sm_%d%d -m%d --cubin \"%s\" "
328                         "-o \"%s\" --ptxas-options=\"-v\" %s -I\"%s\" -DNVCC -D__KERNEL_CUDA_VERSION__=%d",
329                         nvcc.c_str(), major, minor, machine, kernel.c_str(), cubin.c_str(), arch_flags.c_str(), include.c_str(), cuda_version);
330
331                 printf("%s\n", command.c_str());
332
333                 if(system(command.c_str()) == -1) {
334                         cuda_error_message("Failed to execute compilation command, see console for details.");
335                         return "";
336                 }
337
338                 /* verify if compilation succeeded */
339                 if(!path_exists(cubin)) {
340                         cuda_error_message("CUDA kernel compilation failed, see console for details.");
341                         return "";
342                 }
343
344                 printf("Kernel compilation finished in %.2lfs.\n", time_dt() - starttime);
345
346                 return cubin;
347         }
348
349         bool load_kernels(bool experimental)
350         {
351                 /* check if cuda init succeeded */
352                 if(cuContext == 0)
353                         return false;
354                 
355                 /* check if GPU is supported with current feature set */
356                 if(!support_device(experimental))
357                         return false;
358
359                 /* get kernel */
360                 string cubin = compile_kernel();
361
362                 if(cubin == "")
363                         return false;
364
365                 /* open module */
366                 cuda_push_context();
367
368                 CUresult result = cuModuleLoad(&cuModule, cubin.c_str());
369                 if(cuda_error_(result, "cuModuleLoad"))
370                         cuda_error_message(string_printf("Failed loading CUDA kernel %s.", cubin.c_str()));
371
372                 cuda_pop_context();
373
374                 return (result == CUDA_SUCCESS);
375         }
376
377         void mem_alloc(device_memory& mem, MemoryType type)
378         {
379                 cuda_push_context();
380                 CUdeviceptr device_pointer;
381                 size_t size = mem.memory_size();
382                 cuda_assert(cuMemAlloc(&device_pointer, size))
383                 mem.device_pointer = (device_ptr)device_pointer;
384                 stats.mem_alloc(size);
385                 cuda_pop_context();
386         }
387
388         void mem_copy_to(device_memory& mem)
389         {
390                 cuda_push_context();
391                 if(mem.device_pointer)
392                         cuda_assert(cuMemcpyHtoD(cuda_device_ptr(mem.device_pointer), (void*)mem.data_pointer, mem.memory_size()))
393                 cuda_pop_context();
394         }
395
396         void mem_copy_from(device_memory& mem, int y, int w, int h, int elem)
397         {
398                 size_t offset = elem*y*w;
399                 size_t size = elem*w*h;
400
401                 cuda_push_context();
402                 if(mem.device_pointer) {
403                         cuda_assert(cuMemcpyDtoH((uchar*)mem.data_pointer + offset,
404                                 (CUdeviceptr)((uchar*)mem.device_pointer + offset), size))
405                 }
406                 else {
407                         memset((char*)mem.data_pointer + offset, 0, size);
408                 }
409                 cuda_pop_context();
410         }
411
412         void mem_zero(device_memory& mem)
413         {
414                 memset((void*)mem.data_pointer, 0, mem.memory_size());
415
416                 cuda_push_context();
417                 if(mem.device_pointer)
418                         cuda_assert(cuMemsetD8(cuda_device_ptr(mem.device_pointer), 0, mem.memory_size()))
419                 cuda_pop_context();
420         }
421
422         void mem_free(device_memory& mem)
423         {
424                 if(mem.device_pointer) {
425                         cuda_push_context();
426                         cuda_assert(cuMemFree(cuda_device_ptr(mem.device_pointer)))
427                         cuda_pop_context();
428
429                         mem.device_pointer = 0;
430
431                         stats.mem_free(mem.memory_size());
432                 }
433         }
434
435         void const_copy_to(const char *name, void *host, size_t size)
436         {
437                 CUdeviceptr mem;
438                 size_t bytes;
439
440                 cuda_push_context();
441                 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule, name))
442                 //assert(bytes == size);
443                 cuda_assert(cuMemcpyHtoD(mem, host, size))
444                 cuda_pop_context();
445         }
446
447         void tex_alloc(const char *name, device_memory& mem, bool interpolation, bool periodic)
448         {
449                 /* determine format */
450                 CUarray_format_enum format;
451                 size_t dsize = datatype_size(mem.data_type);
452                 size_t size = mem.memory_size();
453
454                 switch(mem.data_type) {
455                         case TYPE_UCHAR: format = CU_AD_FORMAT_UNSIGNED_INT8; break;
456                         case TYPE_UINT: format = CU_AD_FORMAT_UNSIGNED_INT32; break;
457                         case TYPE_INT: format = CU_AD_FORMAT_SIGNED_INT32; break;
458                         case TYPE_FLOAT: format = CU_AD_FORMAT_FLOAT; break;
459                         default: assert(0); return;
460                 }
461
462                 CUtexref texref = NULL;
463
464                 cuda_push_context();
465                 cuda_assert(cuModuleGetTexRef(&texref, cuModule, name))
466
467                 if(!texref) {
468                         cuda_pop_context();
469                         return;
470                 }
471
472                 if(interpolation) {
473                         CUarray handle = NULL;
474                         CUDA_ARRAY_DESCRIPTOR desc;
475
476                         desc.Width = mem.data_width;
477                         desc.Height = mem.data_height;
478                         desc.Format = format;
479                         desc.NumChannels = mem.data_elements;
480
481                         cuda_assert(cuArrayCreate(&handle, &desc))
482
483                         if(!handle) {
484                                 cuda_pop_context();
485                                 return;
486                         }
487
488                         if(mem.data_height > 1) {
489                                 CUDA_MEMCPY2D param;
490                                 memset(&param, 0, sizeof(param));
491                                 param.dstMemoryType = CU_MEMORYTYPE_ARRAY;
492                                 param.dstArray = handle;
493                                 param.srcMemoryType = CU_MEMORYTYPE_HOST;
494                                 param.srcHost = (void*)mem.data_pointer;
495                                 param.srcPitch = mem.data_width*dsize*mem.data_elements;
496                                 param.WidthInBytes = param.srcPitch;
497                                 param.Height = mem.data_height;
498
499                                 cuda_assert(cuMemcpy2D(&param))
500                         }
501                         else
502                                 cuda_assert(cuMemcpyHtoA(handle, 0, (void*)mem.data_pointer, size))
503
504                         cuda_assert(cuTexRefSetArray(texref, handle, CU_TRSA_OVERRIDE_FORMAT))
505
506                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_LINEAR))
507                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_NORMALIZED_COORDINATES))
508
509                         mem.device_pointer = (device_ptr)handle;
510
511                         stats.mem_alloc(size);
512                 }
513                 else {
514                         cuda_pop_context();
515
516                         mem_alloc(mem, MEM_READ_ONLY);
517                         mem_copy_to(mem);
518
519                         cuda_push_context();
520
521                         cuda_assert(cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size))
522                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_POINT))
523                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_READ_AS_INTEGER))
524                 }
525
526                 if(periodic) {
527                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_WRAP))
528                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_WRAP))
529                 }
530                 else {
531                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_CLAMP))
532                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_CLAMP))
533                 }
534                 cuda_assert(cuTexRefSetFormat(texref, format, mem.data_elements))
535
536                 cuda_pop_context();
537
538                 tex_interp_map[mem.device_pointer] = interpolation;
539         }
540
541         void tex_free(device_memory& mem)
542         {
543                 if(mem.device_pointer) {
544                         if(tex_interp_map[mem.device_pointer]) {
545                                 cuda_push_context();
546                                 cuArrayDestroy((CUarray)mem.device_pointer);
547                                 cuda_pop_context();
548
549                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
550                                 mem.device_pointer = 0;
551
552                                 stats.mem_free(mem.memory_size());
553                         }
554                         else {
555                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
556                                 mem_free(mem);
557                         }
558                 }
559         }
560
561         void path_trace(RenderTile& rtile, int sample)
562         {
563                 if(have_error())
564                         return;
565
566                 cuda_push_context();
567
568                 CUfunction cuPathTrace;
569                 CUdeviceptr d_buffer = cuda_device_ptr(rtile.buffer);
570                 CUdeviceptr d_rng_state = cuda_device_ptr(rtile.rng_state);
571
572                 /* get kernel function */
573                 cuda_assert(cuModuleGetFunction(&cuPathTrace, cuModule, "kernel_cuda_path_trace"))
574                 
575                 /* pass in parameters */
576                 int offset = 0;
577                 
578                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_buffer, sizeof(d_buffer)))
579                 offset += sizeof(d_buffer);
580
581                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_rng_state, sizeof(d_rng_state)))
582                 offset += sizeof(d_rng_state);
583
584                 offset = align_up(offset, __alignof(sample));
585
586                 cuda_assert(cuParamSeti(cuPathTrace, offset, sample))
587                 offset += sizeof(sample);
588
589                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.x))
590                 offset += sizeof(rtile.x);
591
592                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.y))
593                 offset += sizeof(rtile.y);
594
595                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.w))
596                 offset += sizeof(rtile.w);
597
598                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.h))
599                 offset += sizeof(rtile.h);
600
601                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.offset))
602                 offset += sizeof(rtile.offset);
603
604                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.stride))
605                 offset += sizeof(rtile.stride);
606
607                 cuda_assert(cuParamSetSize(cuPathTrace, offset))
608
609                 /* launch kernel: todo find optimal size, cache config for fermi */
610                 int xthreads = 16;
611                 int ythreads = 16;
612                 int xblocks = (rtile.w + xthreads - 1)/xthreads;
613                 int yblocks = (rtile.h + ythreads - 1)/ythreads;
614
615                 cuda_assert(cuFuncSetCacheConfig(cuPathTrace, CU_FUNC_CACHE_PREFER_L1))
616                 cuda_assert(cuFuncSetBlockShape(cuPathTrace, xthreads, ythreads, 1))
617                 cuda_assert(cuLaunchGrid(cuPathTrace, xblocks, yblocks))
618
619                 cuda_assert(cuCtxSynchronize())
620
621                 cuda_pop_context();
622         }
623
624         void tonemap(DeviceTask& task, device_ptr buffer, device_ptr rgba)
625         {
626                 if(have_error())
627                         return;
628
629                 cuda_push_context();
630
631                 CUfunction cuFilmConvert;
632                 CUdeviceptr d_rgba = map_pixels(rgba);
633                 CUdeviceptr d_buffer = cuda_device_ptr(buffer);
634
635                 /* get kernel function */
636                 cuda_assert(cuModuleGetFunction(&cuFilmConvert, cuModule, "kernel_cuda_tonemap"))
637
638                 /* pass in parameters */
639                 int offset = 0;
640
641                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_rgba, sizeof(d_rgba)))
642                 offset += sizeof(d_rgba);
643                 
644                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_buffer, sizeof(d_buffer)))
645                 offset += sizeof(d_buffer);
646
647                 int sample = task.sample;
648                 offset = align_up(offset, __alignof(sample));
649
650                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.sample))
651                 offset += sizeof(task.sample);
652
653                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.x))
654                 offset += sizeof(task.x);
655
656                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.y))
657                 offset += sizeof(task.y);
658
659                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.w))
660                 offset += sizeof(task.w);
661
662                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.h))
663                 offset += sizeof(task.h);
664
665                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.offset))
666                 offset += sizeof(task.offset);
667
668                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.stride))
669                 offset += sizeof(task.stride);
670
671                 cuda_assert(cuParamSetSize(cuFilmConvert, offset))
672
673                 /* launch kernel: todo find optimal size, cache config for fermi */
674                 int xthreads = 16;
675                 int ythreads = 16;
676                 int xblocks = (task.w + xthreads - 1)/xthreads;
677                 int yblocks = (task.h + ythreads - 1)/ythreads;
678
679                 cuda_assert(cuFuncSetCacheConfig(cuFilmConvert, CU_FUNC_CACHE_PREFER_L1))
680                 cuda_assert(cuFuncSetBlockShape(cuFilmConvert, xthreads, ythreads, 1))
681                 cuda_assert(cuLaunchGrid(cuFilmConvert, xblocks, yblocks))
682
683                 unmap_pixels(task.rgba);
684
685                 cuda_pop_context();
686         }
687
688         void shader(DeviceTask& task)
689         {
690                 if(have_error())
691                         return;
692
693                 cuda_push_context();
694
695                 CUfunction cuDisplace;
696                 CUdeviceptr d_input = cuda_device_ptr(task.shader_input);
697                 CUdeviceptr d_output = cuda_device_ptr(task.shader_output);
698
699                 /* get kernel function */
700                 cuda_assert(cuModuleGetFunction(&cuDisplace, cuModule, "kernel_cuda_shader"))
701                 
702                 /* pass in parameters */
703                 int offset = 0;
704                 
705                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_input, sizeof(d_input)))
706                 offset += sizeof(d_input);
707
708                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_output, sizeof(d_output)))
709                 offset += sizeof(d_output);
710
711                 int shader_eval_type = task.shader_eval_type;
712                 offset = align_up(offset, __alignof(shader_eval_type));
713
714                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_eval_type))
715                 offset += sizeof(task.shader_eval_type);
716
717                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_x))
718                 offset += sizeof(task.shader_x);
719
720                 cuda_assert(cuParamSetSize(cuDisplace, offset))
721
722                 /* launch kernel: todo find optimal size, cache config for fermi */
723                 int xthreads = 16;
724                 int xblocks = (task.shader_w + xthreads - 1)/xthreads;
725
726                 cuda_assert(cuFuncSetCacheConfig(cuDisplace, CU_FUNC_CACHE_PREFER_L1))
727                 cuda_assert(cuFuncSetBlockShape(cuDisplace, xthreads, 1, 1))
728                 cuda_assert(cuLaunchGrid(cuDisplace, xblocks, 1))
729
730                 cuda_pop_context();
731         }
732
733         CUdeviceptr map_pixels(device_ptr mem)
734         {
735                 if(!background) {
736                         PixelMem pmem = pixel_mem_map[mem];
737                         CUdeviceptr buffer;
738                         
739                         size_t bytes;
740                         cuda_assert(cuGraphicsMapResources(1, &pmem.cuPBOresource, 0))
741                         cuda_assert(cuGraphicsResourceGetMappedPointer(&buffer, &bytes, pmem.cuPBOresource))
742                         
743                         return buffer;
744                 }
745
746                 return cuda_device_ptr(mem);
747         }
748
749         void unmap_pixels(device_ptr mem)
750         {
751                 if(!background) {
752                         PixelMem pmem = pixel_mem_map[mem];
753
754                         cuda_assert(cuGraphicsUnmapResources(1, &pmem.cuPBOresource, 0))
755                 }
756         }
757
758         void pixels_alloc(device_memory& mem)
759         {
760                 if(!background) {
761                         PixelMem pmem;
762
763                         pmem.w = mem.data_width;
764                         pmem.h = mem.data_height;
765
766                         cuda_push_context();
767
768                         glGenBuffers(1, &pmem.cuPBO);
769                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
770                         glBufferData(GL_PIXEL_UNPACK_BUFFER, pmem.w*pmem.h*sizeof(GLfloat)*3, NULL, GL_DYNAMIC_DRAW);
771                         
772                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
773                         
774                         glGenTextures(1, &pmem.cuTexId);
775                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
776                         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, pmem.w, pmem.h, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
777                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
778                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
779                         glBindTexture(GL_TEXTURE_2D, 0);
780                         
781                         CUresult result = cuGraphicsGLRegisterBuffer(&pmem.cuPBOresource, pmem.cuPBO, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
782
783                         if(result == CUDA_SUCCESS) {
784                                 cuda_pop_context();
785
786                                 mem.device_pointer = pmem.cuTexId;
787                                 pixel_mem_map[mem.device_pointer] = pmem;
788
789                                 stats.mem_alloc(mem.memory_size());
790
791                                 return;
792                         }
793                         else {
794                                 /* failed to register buffer, fallback to no interop */
795                                 glDeleteBuffers(1, &pmem.cuPBO);
796                                 glDeleteTextures(1, &pmem.cuTexId);
797
798                                 cuda_pop_context();
799
800                                 background = true;
801                         }
802                 }
803
804                 Device::pixels_alloc(mem);
805         }
806
807         void pixels_copy_from(device_memory& mem, int y, int w, int h)
808         {
809                 if(!background) {
810                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
811
812                         cuda_push_context();
813
814                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
815                         uchar *pixels = (uchar*)glMapBuffer(GL_PIXEL_UNPACK_BUFFER, GL_READ_ONLY);
816                         size_t offset = sizeof(uchar)*4*y*w;
817                         memcpy((uchar*)mem.data_pointer + offset, pixels + offset, sizeof(uchar)*4*w*h);
818                         glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
819                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
820
821                         cuda_pop_context();
822
823                         return;
824                 }
825
826                 Device::pixels_copy_from(mem, y, w, h);
827         }
828
829         void pixels_free(device_memory& mem)
830         {
831                 if(mem.device_pointer) {
832                         if(!background) {
833                                 PixelMem pmem = pixel_mem_map[mem.device_pointer];
834
835                                 cuda_push_context();
836
837                                 cuda_assert(cuGraphicsUnregisterResource(pmem.cuPBOresource))
838                                 glDeleteBuffers(1, &pmem.cuPBO);
839                                 glDeleteTextures(1, &pmem.cuTexId);
840
841                                 cuda_pop_context();
842
843                                 pixel_mem_map.erase(pixel_mem_map.find(mem.device_pointer));
844                                 mem.device_pointer = 0;
845
846                                 stats.mem_free(mem.memory_size());
847
848                                 return;
849                         }
850
851                         Device::pixels_free(mem);
852                 }
853         }
854
855         void draw_pixels(device_memory& mem, int y, int w, int h, int dy, int width, int height, bool transparent)
856         {
857                 if(!background) {
858                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
859
860                         cuda_push_context();
861
862                         /* for multi devices, this assumes the ineffecient method that we allocate
863                          * all pixels on the device even though we only render to a subset */
864                         size_t offset = sizeof(uint8_t)*4*y*w;
865
866                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pmem.cuPBO);
867                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
868                         glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, (void*)offset);
869                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
870                         
871                         glEnable(GL_TEXTURE_2D);
872                         
873                         if(transparent) {
874                                 glEnable(GL_BLEND);
875                                 glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
876                         }
877
878                         glColor3f(1.0f, 1.0f, 1.0f);
879
880                         glPushMatrix();
881                         glTranslatef(0.0f, (float)dy, 0.0f);
882                                 
883                         glBegin(GL_QUADS);
884                         
885                         glTexCoord2f(0.0f, 0.0f);
886                         glVertex2f(0.0f, 0.0f);
887                         glTexCoord2f((float)w/(float)pmem.w, 0.0f);
888                         glVertex2f((float)width, 0.0f);
889                         glTexCoord2f((float)w/(float)pmem.w, (float)h/(float)pmem.h);
890                         glVertex2f((float)width, (float)height);
891                         glTexCoord2f(0.0f, (float)h/(float)pmem.h);
892                         glVertex2f(0.0f, (float)height);
893
894                         glEnd();
895
896                         glPopMatrix();
897
898                         if(transparent)
899                                 glDisable(GL_BLEND);
900                         
901                         glBindTexture(GL_TEXTURE_2D, 0);
902                         glDisable(GL_TEXTURE_2D);
903
904                         cuda_pop_context();
905
906                         return;
907                 }
908
909                 Device::draw_pixels(mem, y, w, h, dy, width, height, transparent);
910         }
911
912         void thread_run(DeviceTask *task)
913         {
914                 if(task->type == DeviceTask::PATH_TRACE) {
915                         RenderTile tile;
916                         
917                         /* keep rendering tiles until done */
918                         while(task->acquire_tile(this, tile)) {
919                                 int start_sample = tile.start_sample;
920                                 int end_sample = tile.start_sample + tile.num_samples;
921
922                                 for(int sample = start_sample; sample < end_sample; sample++) {
923                                         if (task->get_cancel()) {
924                                                 if(task->need_finish_queue == false)
925                                                         break;
926                                         }
927
928                                         path_trace(tile, sample);
929
930                                         tile.sample = sample + 1;
931
932                                         task->update_progress(tile);
933                                 }
934
935                                 task->release_tile(tile);
936                         }
937                 }
938                 else if(task->type == DeviceTask::SHADER) {
939                         shader(*task);
940
941                         cuda_push_context();
942                         cuda_assert(cuCtxSynchronize())
943                         cuda_pop_context();
944                 }
945         }
946
947         class CUDADeviceTask : public DeviceTask {
948         public:
949                 CUDADeviceTask(CUDADevice *device, DeviceTask& task)
950                 : DeviceTask(task)
951                 {
952                         run = function_bind(&CUDADevice::thread_run, device, this);
953                 }
954         };
955
956         void task_add(DeviceTask& task)
957         {
958                 if(task.type == DeviceTask::TONEMAP) {
959                         /* must be done in main thread due to opengl access */
960                         tonemap(task, task.buffer, task.rgba);
961
962                         cuda_push_context();
963                         cuda_assert(cuCtxSynchronize())
964                         cuda_pop_context();
965                 }
966                 else {
967                         task_pool.push(new CUDADeviceTask(this, task));
968                 }
969         }
970
971         void task_wait()
972         {
973                 task_pool.wait_work();
974         }
975
976         void task_cancel()
977         {
978                 task_pool.cancel();
979         }
980 };
981
982 Device *device_cuda_create(DeviceInfo& info, Stats &stats, bool background)
983 {
984         return new CUDADevice(info, stats, background);
985 }
986
987 void device_cuda_info(vector<DeviceInfo>& devices)
988 {
989         CUresult result;
990         int count = 0;
991
992         result = cuInit(0);
993         if(result != CUDA_SUCCESS) {
994                 if(result != CUDA_ERROR_NO_DEVICE)
995                         fprintf(stderr, "CUDA cuInit: %s\n", CUDADevice::cuda_error_string(result));
996                 return;
997         }
998
999         result = cuDeviceGetCount(&count);
1000         if(result != CUDA_SUCCESS) {
1001                 fprintf(stderr, "CUDA cuDeviceGetCount: %s\n", CUDADevice::cuda_error_string(result));
1002                 return;
1003         }
1004         
1005         vector<DeviceInfo> display_devices;
1006         
1007         for(int num = 0; num < count; num++) {
1008                 char name[256];
1009                 int attr;
1010                 
1011                 if(cuDeviceGetName(name, 256, num) != CUDA_SUCCESS)
1012                         continue;
1013
1014                 DeviceInfo info;
1015
1016                 info.type = DEVICE_CUDA;
1017                 info.description = string(name);
1018                 info.id = string_printf("CUDA_%d", num);
1019                 info.num = num;
1020
1021                 int major, minor;
1022                 cuDeviceComputeCapability(&major, &minor, num);
1023                 info.advanced_shading = (major >= 2);
1024                 info.extended_images = (major >= 3);
1025                 info.pack_images = false;
1026
1027                 /* if device has a kernel timeout, assume it is used for display */
1028                 if(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, num) == CUDA_SUCCESS && attr == 1) {
1029                         info.display_device = true;
1030                         display_devices.push_back(info);
1031                 }
1032                 else
1033                         devices.push_back(info);
1034         }
1035
1036         if(!display_devices.empty())
1037                 devices.insert(devices.end(), display_devices.begin(), display_devices.end());
1038 }
1039
1040 CCL_NAMESPACE_END
1041