Merged revision(s) 57587-57670 from trunk/blender into soc-2013-dingto
[blender.git] / intern / cycles / device / device_cuda.cpp
1 /*
2  * Copyright 2011, Blender Foundation.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  */
18
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <string.h>
22
23 #include "device.h"
24 #include "device_intern.h"
25
26 #include "buffers.h"
27
28 #include "util_cuda.h"
29 #include "util_debug.h"
30 #include "util_map.h"
31 #include "util_opengl.h"
32 #include "util_path.h"
33 #include "util_system.h"
34 #include "util_types.h"
35 #include "util_time.h"
36
37 CCL_NAMESPACE_BEGIN
38
39 class CUDADevice : public Device
40 {
41 public:
42         TaskPool task_pool;
43         CUdevice cuDevice;
44         CUcontext cuContext;
45         CUmodule cuModule;
46         map<device_ptr, bool> tex_interp_map;
47         int cuDevId;
48         bool first_error;
49
50         struct PixelMem {
51                 GLuint cuPBO;
52                 CUgraphicsResource cuPBOresource;
53                 GLuint cuTexId;
54                 int w, h;
55         };
56
57         map<device_ptr, PixelMem> pixel_mem_map;
58
59         CUdeviceptr cuda_device_ptr(device_ptr mem)
60         {
61                 return (CUdeviceptr)mem;
62         }
63
64         static const char *cuda_error_string(CUresult result)
65         {
66                 switch(result) {
67                         case CUDA_SUCCESS: return "No errors";
68                         case CUDA_ERROR_INVALID_VALUE: return "Invalid value";
69                         case CUDA_ERROR_OUT_OF_MEMORY: return "Out of memory";
70                         case CUDA_ERROR_NOT_INITIALIZED: return "Driver not initialized";
71                         case CUDA_ERROR_DEINITIALIZED: return "Driver deinitialized";
72
73                         case CUDA_ERROR_NO_DEVICE: return "No CUDA-capable device available";
74                         case CUDA_ERROR_INVALID_DEVICE: return "Invalid device";
75
76                         case CUDA_ERROR_INVALID_IMAGE: return "Invalid kernel image";
77                         case CUDA_ERROR_INVALID_CONTEXT: return "Invalid context";
78                         case CUDA_ERROR_CONTEXT_ALREADY_CURRENT: return "Context already current";
79                         case CUDA_ERROR_MAP_FAILED: return "Map failed";
80                         case CUDA_ERROR_UNMAP_FAILED: return "Unmap failed";
81                         case CUDA_ERROR_ARRAY_IS_MAPPED: return "Array is mapped";
82                         case CUDA_ERROR_ALREADY_MAPPED: return "Already mapped";
83                         case CUDA_ERROR_NO_BINARY_FOR_GPU: return "No binary for GPU";
84                         case CUDA_ERROR_ALREADY_ACQUIRED: return "Already acquired";
85                         case CUDA_ERROR_NOT_MAPPED: return "Not mapped";
86                         case CUDA_ERROR_NOT_MAPPED_AS_ARRAY: return "Mapped resource not available for access as an array";
87                         case CUDA_ERROR_NOT_MAPPED_AS_POINTER: return "Mapped resource not available for access as a pointer";
88                         case CUDA_ERROR_ECC_UNCORRECTABLE: return "Uncorrectable ECC error detected";
89                         case CUDA_ERROR_UNSUPPORTED_LIMIT: return "CUlimit not supported by device";
90
91                         case CUDA_ERROR_INVALID_SOURCE: return "Invalid source";
92                         case CUDA_ERROR_FILE_NOT_FOUND: return "File not found";
93                         case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND: return "Link to a shared object failed to resolve";
94                         case CUDA_ERROR_SHARED_OBJECT_INIT_FAILED: return "Shared object initialization failed";
95
96                         case CUDA_ERROR_INVALID_HANDLE: return "Invalid handle";
97
98                         case CUDA_ERROR_NOT_FOUND: return "Not found";
99
100                         case CUDA_ERROR_NOT_READY: return "CUDA not ready";
101
102                         case CUDA_ERROR_LAUNCH_FAILED: return "Launch failed";
103                         case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES: return "Launch exceeded resources";
104                         case CUDA_ERROR_LAUNCH_TIMEOUT: return "Launch exceeded timeout";
105                         case CUDA_ERROR_LAUNCH_INCOMPATIBLE_TEXTURING: return "Launch with incompatible texturing";
106
107                         case CUDA_ERROR_UNKNOWN: return "Unknown error";
108
109                         default: return "Unknown CUDA error value";
110                 }
111         }
112
113 /*#ifdef NDEBUG
114 #define cuda_abort()
115 #else
116 #define cuda_abort() abort()
117 #endif*/
118         void cuda_error_documentation()
119         {
120                 if(first_error) {
121                         fprintf(stderr, "\nRefer to the Cycles GPU rendering documentation for possible solutions:\n");
122                         fprintf(stderr, "http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/GPU_Rendering\n\n");
123                         first_error = false;
124                 }
125         }
126
127 #define cuda_assert(stmt) \
128         { \
129                 CUresult result = stmt; \
130                 \
131                 if(result != CUDA_SUCCESS) { \
132                         string message = string_printf("CUDA error: %s in %s", cuda_error_string(result), #stmt); \
133                         if(error_msg == "") \
134                                 error_msg = message; \
135                         fprintf(stderr, "%s\n", message.c_str()); \
136                         /*cuda_abort();*/ \
137                         cuda_error_documentation(); \
138                 } \
139         }
140
141         bool cuda_error_(CUresult result, const string& stmt)
142         {
143                 if(result == CUDA_SUCCESS)
144                         return false;
145
146                 string message = string_printf("CUDA error at %s: %s", stmt.c_str(), cuda_error_string(result));
147                 if(error_msg == "")
148                         error_msg = message;
149                 fprintf(stderr, "%s\n", message.c_str());
150                 cuda_error_documentation();
151                 return true;
152         }
153
154 #define cuda_error(stmt) cuda_error_(stmt, #stmt)
155
156         void cuda_error_message(const string& message)
157         {
158                 if(error_msg == "")
159                         error_msg = message;
160                 fprintf(stderr, "%s\n", message.c_str());
161                 cuda_error_documentation();
162         }
163
164         void cuda_push_context()
165         {
166                 cuda_assert(cuCtxSetCurrent(cuContext))
167         }
168
169         void cuda_pop_context()
170         {
171                 cuda_assert(cuCtxSetCurrent(NULL));
172         }
173
174         CUDADevice(DeviceInfo& info, Stats &stats, bool background_) : Device(stats)
175         {
176                 first_error = true;
177                 background = background_;
178
179                 cuDevId = info.num;
180                 cuDevice = 0;
181                 cuContext = 0;
182
183                 /* intialize */
184                 if(cuda_error(cuInit(0)))
185                         return;
186
187                 /* setup device and context */
188                 if(cuda_error(cuDeviceGet(&cuDevice, cuDevId)))
189                         return;
190
191                 CUresult result;
192
193                 if(background) {
194                         result = cuCtxCreate(&cuContext, 0, cuDevice);
195                 }
196                 else {
197                         result = cuGLCtxCreate(&cuContext, 0, cuDevice);
198
199                         if(result != CUDA_SUCCESS) {
200                                 result = cuCtxCreate(&cuContext, 0, cuDevice);
201                                 background = true;
202                         }
203                 }
204
205                 if(cuda_error_(result, "cuCtxCreate"))
206                         return;
207
208                 cuda_pop_context();
209         }
210
211         ~CUDADevice()
212         {
213                 task_pool.stop();
214
215                 cuda_push_context();
216                 cuda_assert(cuCtxDetach(cuContext))
217         }
218
219         bool support_device(bool experimental)
220         {
221                 if(!experimental) {
222                         int major, minor;
223                         cuDeviceComputeCapability(&major, &minor, cuDevId);
224
225                         if(major < 2) {
226                                 cuda_error_message(string_printf("CUDA device supported only with compute capability 2.0 or up, found %d.%d.", major, minor));
227                                 return false;
228                         }
229                 }
230
231                 return true;
232         }
233
234         string compile_kernel()
235         {
236                 /* compute cubin name */
237                 int major, minor;
238                 cuDeviceComputeCapability(&major, &minor, cuDevId);
239
240                 /* attempt to use kernel provided with blender */
241                 string cubin = path_get(string_printf("lib/kernel_sm_%d%d.cubin", major, minor));
242                 if(path_exists(cubin))
243                         return cubin;
244
245                 /* not found, try to use locally compiled kernel */
246                 string kernel_path = path_get("kernel");
247                 string md5 = path_files_md5_hash(kernel_path);
248
249                 cubin = string_printf("cycles_kernel_sm%d%d_%s.cubin", major, minor, md5.c_str());
250                 cubin = path_user_get(path_join("cache", cubin));
251
252                 /* if exists already, use it */
253                 if(path_exists(cubin))
254                         return cubin;
255
256 #ifdef _WIN32
257                 if(cuHavePrecompiledKernels()) {
258                         if(major < 2)
259                                 cuda_error_message(string_printf("CUDA device requires compute capability 2.0 or up, found %d.%d. Your GPU is not supported.", major, minor));
260                         else
261                                 cuda_error_message(string_printf("CUDA binary kernel for this graphics card compute capability (%d.%d) not found.", major, minor));
262                         return "";
263                 }
264 #endif
265
266                 /* if not, find CUDA compiler */
267                 string nvcc = cuCompilerPath();
268
269                 if(nvcc == "") {
270                         cuda_error_message("CUDA nvcc compiler not found. Install CUDA toolkit in default location.");
271                         return "";
272                 }
273
274                 int cuda_version = cuCompilerVersion();
275
276                 if(cuda_version == 0) {
277                         cuda_error_message("CUDA nvcc compiler version could not be parsed.");
278                         return "";
279                 }
280
281                 if(cuda_version != 50)
282                         printf("CUDA version %d.%d detected, build may succeed but only CUDA 5.0 is officially supported.\n", cuda_version/10, cuda_version%10);
283
284                 /* compile */
285                 string kernel = path_join(kernel_path, "kernel.cu");
286                 string include = kernel_path;
287                 const int machine = system_cpu_bits();
288                 string arch_flags;
289
290                 /* build flags depending on CUDA version and arch */
291                 if(cuda_version < 50) {
292                         /* CUDA 4.x */
293                         if(major == 1) {
294                                 /* sm_1x */
295                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0";
296                         }
297                         else if(major == 2) {
298                                 /* sm_2x */
299                                 arch_flags = "--maxrregcount=24";
300                         }
301                         else {
302                                 /* sm_3x */
303                                 arch_flags = "--maxrregcount=32";
304                         }
305                 }
306                 else {
307                         /* CUDA 5.x */
308                         if(major == 1) {
309                                 /* sm_1x */
310                                 arch_flags = "--maxrregcount=24 --opencc-options -OPT:Olimit=0 --use_fast_math";
311                         }
312                         else if(major == 2) {
313                                 /* sm_2x */
314                                 arch_flags = "--maxrregcount=32 --use_fast_math";
315                         }
316                         else {
317                                 /* sm_3x */
318                                 arch_flags = "--maxrregcount=32 --use_fast_math";
319                         }
320                 }
321
322                 double starttime = time_dt();
323                 printf("Compiling CUDA kernel ...\n");
324
325                 path_create_directories(cubin);
326
327                 string command = string_printf("\"%s\" -arch=sm_%d%d -m%d --cubin \"%s\" "
328                         "-o \"%s\" --ptxas-options=\"-v\" %s -I\"%s\" -DNVCC -D__KERNEL_CUDA_VERSION__=%d",
329                         nvcc.c_str(), major, minor, machine, kernel.c_str(), cubin.c_str(), arch_flags.c_str(), include.c_str(), cuda_version);
330
331                 printf("%s\n", command.c_str());
332
333                 if(system(command.c_str()) == -1) {
334                         cuda_error_message("Failed to execute compilation command, see console for details.");
335                         return "";
336                 }
337
338                 /* verify if compilation succeeded */
339                 if(!path_exists(cubin)) {
340                         cuda_error_message("CUDA kernel compilation failed, see console for details.");
341                         return "";
342                 }
343
344                 printf("Kernel compilation finished in %.2lfs.\n", time_dt() - starttime);
345
346                 return cubin;
347         }
348
349         bool load_kernels(bool experimental)
350         {
351                 /* check if cuda init succeeded */
352                 if(cuContext == 0)
353                         return false;
354                 
355                 /* check if GPU is supported with current feature set */
356                 if(!support_device(experimental))
357                         return false;
358
359                 /* get kernel */
360                 string cubin = compile_kernel();
361
362                 if(cubin == "")
363                         return false;
364
365                 /* open module */
366                 cuda_push_context();
367
368                 CUresult result = cuModuleLoad(&cuModule, cubin.c_str());
369                 if(cuda_error_(result, "cuModuleLoad"))
370                         cuda_error_message(string_printf("Failed loading CUDA kernel %s.", cubin.c_str()));
371
372                 cuda_pop_context();
373
374                 return (result == CUDA_SUCCESS);
375         }
376
377         void mem_alloc(device_memory& mem, MemoryType type)
378         {
379                 cuda_push_context();
380                 CUdeviceptr device_pointer;
381                 size_t size = mem.memory_size();
382                 cuda_assert(cuMemAlloc(&device_pointer, size))
383                 mem.device_pointer = (device_ptr)device_pointer;
384                 stats.mem_alloc(size);
385                 cuda_pop_context();
386         }
387
388         void mem_copy_to(device_memory& mem)
389         {
390                 cuda_push_context();
391                 if(mem.device_pointer)
392                         cuda_assert(cuMemcpyHtoD(cuda_device_ptr(mem.device_pointer), (void*)mem.data_pointer, mem.memory_size()))
393                 cuda_pop_context();
394         }
395
396         void mem_copy_from(device_memory& mem, int y, int w, int h, int elem)
397         {
398                 size_t offset = elem*y*w;
399                 size_t size = elem*w*h;
400
401                 cuda_push_context();
402                 if(mem.device_pointer) {
403                         cuda_assert(cuMemcpyDtoH((uchar*)mem.data_pointer + offset,
404                                 (CUdeviceptr)((uchar*)mem.device_pointer + offset), size))
405                 }
406                 else {
407                         memset((char*)mem.data_pointer + offset, 0, size);
408                 }
409                 cuda_pop_context();
410         }
411
412         void mem_zero(device_memory& mem)
413         {
414                 memset((void*)mem.data_pointer, 0, mem.memory_size());
415
416                 cuda_push_context();
417                 if(mem.device_pointer)
418                         cuda_assert(cuMemsetD8(cuda_device_ptr(mem.device_pointer), 0, mem.memory_size()))
419                 cuda_pop_context();
420         }
421
422         void mem_free(device_memory& mem)
423         {
424                 if(mem.device_pointer) {
425                         cuda_push_context();
426                         cuda_assert(cuMemFree(cuda_device_ptr(mem.device_pointer)))
427                         cuda_pop_context();
428
429                         mem.device_pointer = 0;
430
431                         stats.mem_free(mem.memory_size());
432                 }
433         }
434
435         void const_copy_to(const char *name, void *host, size_t size)
436         {
437                 CUdeviceptr mem;
438                 size_t bytes;
439
440                 cuda_push_context();
441                 cuda_assert(cuModuleGetGlobal(&mem, &bytes, cuModule, name))
442                 //assert(bytes == size);
443                 cuda_assert(cuMemcpyHtoD(mem, host, size))
444                 cuda_pop_context();
445         }
446
447         void tex_alloc(const char *name, device_memory& mem, bool interpolation, bool periodic)
448         {
449                 /* determine format */
450                 CUarray_format_enum format;
451                 size_t dsize = datatype_size(mem.data_type);
452                 size_t size = mem.memory_size();
453
454                 switch(mem.data_type) {
455                         case TYPE_UCHAR: format = CU_AD_FORMAT_UNSIGNED_INT8; break;
456                         case TYPE_UINT: format = CU_AD_FORMAT_UNSIGNED_INT32; break;
457                         case TYPE_INT: format = CU_AD_FORMAT_SIGNED_INT32; break;
458                         case TYPE_FLOAT: format = CU_AD_FORMAT_FLOAT; break;
459                         default: assert(0); return;
460                 }
461
462                 CUtexref texref = NULL;
463
464                 cuda_push_context();
465                 cuda_assert(cuModuleGetTexRef(&texref, cuModule, name))
466
467                 if(!texref) {
468                         cuda_pop_context();
469                         return;
470                 }
471
472                 if(interpolation) {
473                         CUarray handle = NULL;
474                         CUDA_ARRAY_DESCRIPTOR desc;
475
476                         desc.Width = mem.data_width;
477                         desc.Height = mem.data_height;
478                         desc.Format = format;
479                         desc.NumChannels = mem.data_elements;
480
481                         cuda_assert(cuArrayCreate(&handle, &desc))
482
483                         if(!handle) {
484                                 cuda_pop_context();
485                                 return;
486                         }
487
488                         if(mem.data_height > 1) {
489                                 CUDA_MEMCPY2D param;
490                                 memset(&param, 0, sizeof(param));
491                                 param.dstMemoryType = CU_MEMORYTYPE_ARRAY;
492                                 param.dstArray = handle;
493                                 param.srcMemoryType = CU_MEMORYTYPE_HOST;
494                                 param.srcHost = (void*)mem.data_pointer;
495                                 param.srcPitch = mem.data_width*dsize*mem.data_elements;
496                                 param.WidthInBytes = param.srcPitch;
497                                 param.Height = mem.data_height;
498
499                                 cuda_assert(cuMemcpy2D(&param))
500                         }
501                         else
502                                 cuda_assert(cuMemcpyHtoA(handle, 0, (void*)mem.data_pointer, size))
503
504                         cuda_assert(cuTexRefSetArray(texref, handle, CU_TRSA_OVERRIDE_FORMAT))
505
506                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_LINEAR))
507                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_NORMALIZED_COORDINATES))
508
509                         mem.device_pointer = (device_ptr)handle;
510
511                         stats.mem_alloc(size);
512                 }
513                 else {
514                         cuda_pop_context();
515
516                         mem_alloc(mem, MEM_READ_ONLY);
517                         mem_copy_to(mem);
518
519                         cuda_push_context();
520
521                         cuda_assert(cuTexRefSetAddress(NULL, texref, cuda_device_ptr(mem.device_pointer), size))
522                         cuda_assert(cuTexRefSetFilterMode(texref, CU_TR_FILTER_MODE_POINT))
523                         cuda_assert(cuTexRefSetFlags(texref, CU_TRSF_READ_AS_INTEGER))
524                 }
525
526                 if(periodic) {
527                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_WRAP))
528                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_WRAP))
529                 }
530                 else {
531                         cuda_assert(cuTexRefSetAddressMode(texref, 0, CU_TR_ADDRESS_MODE_CLAMP))
532                         cuda_assert(cuTexRefSetAddressMode(texref, 1, CU_TR_ADDRESS_MODE_CLAMP))
533                 }
534                 cuda_assert(cuTexRefSetFormat(texref, format, mem.data_elements))
535
536                 cuda_pop_context();
537
538                 tex_interp_map[mem.device_pointer] = interpolation;
539         }
540
541         void tex_free(device_memory& mem)
542         {
543                 if(mem.device_pointer) {
544                         if(tex_interp_map[mem.device_pointer]) {
545                                 cuda_push_context();
546                                 cuArrayDestroy((CUarray)mem.device_pointer);
547                                 cuda_pop_context();
548
549                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
550                                 mem.device_pointer = 0;
551
552                                 stats.mem_free(mem.memory_size());
553                         }
554                         else {
555                                 tex_interp_map.erase(tex_interp_map.find(mem.device_pointer));
556                                 mem_free(mem);
557                         }
558                 }
559         }
560
561         void path_trace(RenderTile& rtile, int sample)
562         {
563                 if(have_error())
564                         return;
565
566                 cuda_push_context();
567
568                 CUfunction cuPathTrace;
569                 CUdeviceptr d_buffer = cuda_device_ptr(rtile.buffer);
570                 CUdeviceptr d_rng_state = cuda_device_ptr(rtile.rng_state);
571
572                 /* get kernel function */
573                 cuda_assert(cuModuleGetFunction(&cuPathTrace, cuModule, "kernel_cuda_path_trace"))
574                 
575                 /* pass in parameters */
576                 int offset = 0;
577                 
578                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_buffer, sizeof(d_buffer)))
579                 offset += sizeof(d_buffer);
580
581                 cuda_assert(cuParamSetv(cuPathTrace, offset, &d_rng_state, sizeof(d_rng_state)))
582                 offset += sizeof(d_rng_state);
583
584                 offset = align_up(offset, __alignof(sample));
585
586                 cuda_assert(cuParamSeti(cuPathTrace, offset, sample))
587                 offset += sizeof(sample);
588
589                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.x))
590                 offset += sizeof(rtile.x);
591
592                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.y))
593                 offset += sizeof(rtile.y);
594
595                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.w))
596                 offset += sizeof(rtile.w);
597
598                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.h))
599                 offset += sizeof(rtile.h);
600
601                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.offset))
602                 offset += sizeof(rtile.offset);
603
604                 cuda_assert(cuParamSeti(cuPathTrace, offset, rtile.stride))
605                 offset += sizeof(rtile.stride);
606
607                 cuda_assert(cuParamSetSize(cuPathTrace, offset))
608
609                 /* launch kernel: todo find optimal size, cache config for fermi */
610 #ifndef __APPLE__
611                 int xthreads = 16;
612                 int ythreads = 16;
613 #else
614                 int xthreads = 8;
615                 int ythreads = 8;
616 #endif
617                 int xblocks = (rtile.w + xthreads - 1)/xthreads;
618                 int yblocks = (rtile.h + ythreads - 1)/ythreads;
619
620                 cuda_assert(cuFuncSetCacheConfig(cuPathTrace, CU_FUNC_CACHE_PREFER_L1))
621                 cuda_assert(cuFuncSetBlockShape(cuPathTrace, xthreads, ythreads, 1))
622                 cuda_assert(cuLaunchGrid(cuPathTrace, xblocks, yblocks))
623
624                 cuda_assert(cuCtxSynchronize())
625
626                 cuda_pop_context();
627         }
628
629         void tonemap(DeviceTask& task, device_ptr buffer, device_ptr rgba)
630         {
631                 if(have_error())
632                         return;
633
634                 cuda_push_context();
635
636                 CUfunction cuFilmConvert;
637                 CUdeviceptr d_rgba = map_pixels(rgba);
638                 CUdeviceptr d_buffer = cuda_device_ptr(buffer);
639
640                 /* get kernel function */
641                 cuda_assert(cuModuleGetFunction(&cuFilmConvert, cuModule, "kernel_cuda_tonemap"))
642
643                 /* pass in parameters */
644                 int offset = 0;
645
646                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_rgba, sizeof(d_rgba)))
647                 offset += sizeof(d_rgba);
648                 
649                 cuda_assert(cuParamSetv(cuFilmConvert, offset, &d_buffer, sizeof(d_buffer)))
650                 offset += sizeof(d_buffer);
651
652                 int sample = task.sample;
653                 offset = align_up(offset, __alignof(sample));
654
655                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.sample))
656                 offset += sizeof(task.sample);
657
658                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.x))
659                 offset += sizeof(task.x);
660
661                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.y))
662                 offset += sizeof(task.y);
663
664                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.w))
665                 offset += sizeof(task.w);
666
667                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.h))
668                 offset += sizeof(task.h);
669
670                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.offset))
671                 offset += sizeof(task.offset);
672
673                 cuda_assert(cuParamSeti(cuFilmConvert, offset, task.stride))
674                 offset += sizeof(task.stride);
675
676                 cuda_assert(cuParamSetSize(cuFilmConvert, offset))
677
678                 /* launch kernel: todo find optimal size, cache config for fermi */
679 #ifndef __APPLE__
680                 int xthreads = 16;
681                 int ythreads = 16;
682 #else
683                 int xthreads = 8;
684                 int ythreads = 8;
685 #endif
686                 int xblocks = (task.w + xthreads - 1)/xthreads;
687                 int yblocks = (task.h + ythreads - 1)/ythreads;
688
689                 cuda_assert(cuFuncSetCacheConfig(cuFilmConvert, CU_FUNC_CACHE_PREFER_L1))
690                 cuda_assert(cuFuncSetBlockShape(cuFilmConvert, xthreads, ythreads, 1))
691                 cuda_assert(cuLaunchGrid(cuFilmConvert, xblocks, yblocks))
692
693                 unmap_pixels(task.rgba);
694
695                 cuda_pop_context();
696         }
697
698         void shader(DeviceTask& task)
699         {
700                 if(have_error())
701                         return;
702
703                 cuda_push_context();
704
705                 CUfunction cuDisplace;
706                 CUdeviceptr d_input = cuda_device_ptr(task.shader_input);
707                 CUdeviceptr d_output = cuda_device_ptr(task.shader_output);
708
709                 /* get kernel function */
710                 cuda_assert(cuModuleGetFunction(&cuDisplace, cuModule, "kernel_cuda_shader"))
711                 
712                 /* pass in parameters */
713                 int offset = 0;
714                 
715                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_input, sizeof(d_input)))
716                 offset += sizeof(d_input);
717
718                 cuda_assert(cuParamSetv(cuDisplace, offset, &d_output, sizeof(d_output)))
719                 offset += sizeof(d_output);
720
721                 int shader_eval_type = task.shader_eval_type;
722                 offset = align_up(offset, __alignof(shader_eval_type));
723
724                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_eval_type))
725                 offset += sizeof(task.shader_eval_type);
726
727                 cuda_assert(cuParamSeti(cuDisplace, offset, task.shader_x))
728                 offset += sizeof(task.shader_x);
729
730                 cuda_assert(cuParamSetSize(cuDisplace, offset))
731
732                 /* launch kernel: todo find optimal size, cache config for fermi */
733 #ifndef __APPLE__
734                 int xthreads = 16;
735 #else
736                 int xthreads = 8;
737 #endif
738                 int xblocks = (task.shader_w + xthreads - 1)/xthreads;
739
740                 cuda_assert(cuFuncSetCacheConfig(cuDisplace, CU_FUNC_CACHE_PREFER_L1))
741                 cuda_assert(cuFuncSetBlockShape(cuDisplace, xthreads, 1, 1))
742                 cuda_assert(cuLaunchGrid(cuDisplace, xblocks, 1))
743
744                 cuda_pop_context();
745         }
746
747         CUdeviceptr map_pixels(device_ptr mem)
748         {
749                 if(!background) {
750                         PixelMem pmem = pixel_mem_map[mem];
751                         CUdeviceptr buffer;
752                         
753                         size_t bytes;
754                         cuda_assert(cuGraphicsMapResources(1, &pmem.cuPBOresource, 0))
755                         cuda_assert(cuGraphicsResourceGetMappedPointer(&buffer, &bytes, pmem.cuPBOresource))
756                         
757                         return buffer;
758                 }
759
760                 return cuda_device_ptr(mem);
761         }
762
763         void unmap_pixels(device_ptr mem)
764         {
765                 if(!background) {
766                         PixelMem pmem = pixel_mem_map[mem];
767
768                         cuda_assert(cuGraphicsUnmapResources(1, &pmem.cuPBOresource, 0))
769                 }
770         }
771
772         void pixels_alloc(device_memory& mem)
773         {
774                 if(!background) {
775                         PixelMem pmem;
776
777                         pmem.w = mem.data_width;
778                         pmem.h = mem.data_height;
779
780                         cuda_push_context();
781
782                         glGenBuffers(1, &pmem.cuPBO);
783                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
784                         glBufferData(GL_PIXEL_UNPACK_BUFFER, pmem.w*pmem.h*sizeof(GLfloat)*3, NULL, GL_DYNAMIC_DRAW);
785                         
786                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
787                         
788                         glGenTextures(1, &pmem.cuTexId);
789                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
790                         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, pmem.w, pmem.h, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
791                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
792                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
793                         glBindTexture(GL_TEXTURE_2D, 0);
794                         
795                         CUresult result = cuGraphicsGLRegisterBuffer(&pmem.cuPBOresource, pmem.cuPBO, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
796
797                         if(result == CUDA_SUCCESS) {
798                                 cuda_pop_context();
799
800                                 mem.device_pointer = pmem.cuTexId;
801                                 pixel_mem_map[mem.device_pointer] = pmem;
802
803                                 stats.mem_alloc(mem.memory_size());
804
805                                 return;
806                         }
807                         else {
808                                 /* failed to register buffer, fallback to no interop */
809                                 glDeleteBuffers(1, &pmem.cuPBO);
810                                 glDeleteTextures(1, &pmem.cuTexId);
811
812                                 cuda_pop_context();
813
814                                 background = true;
815                         }
816                 }
817
818                 Device::pixels_alloc(mem);
819         }
820
821         void pixels_copy_from(device_memory& mem, int y, int w, int h)
822         {
823                 if(!background) {
824                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
825
826                         cuda_push_context();
827
828                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pmem.cuPBO);
829                         uchar *pixels = (uchar*)glMapBuffer(GL_PIXEL_UNPACK_BUFFER, GL_READ_ONLY);
830                         size_t offset = sizeof(uchar)*4*y*w;
831                         memcpy((uchar*)mem.data_pointer + offset, pixels + offset, sizeof(uchar)*4*w*h);
832                         glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
833                         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
834
835                         cuda_pop_context();
836
837                         return;
838                 }
839
840                 Device::pixels_copy_from(mem, y, w, h);
841         }
842
843         void pixels_free(device_memory& mem)
844         {
845                 if(mem.device_pointer) {
846                         if(!background) {
847                                 PixelMem pmem = pixel_mem_map[mem.device_pointer];
848
849                                 cuda_push_context();
850
851                                 cuda_assert(cuGraphicsUnregisterResource(pmem.cuPBOresource))
852                                 glDeleteBuffers(1, &pmem.cuPBO);
853                                 glDeleteTextures(1, &pmem.cuTexId);
854
855                                 cuda_pop_context();
856
857                                 pixel_mem_map.erase(pixel_mem_map.find(mem.device_pointer));
858                                 mem.device_pointer = 0;
859
860                                 stats.mem_free(mem.memory_size());
861
862                                 return;
863                         }
864
865                         Device::pixels_free(mem);
866                 }
867         }
868
869         void draw_pixels(device_memory& mem, int y, int w, int h, int dy, int width, int height, bool transparent)
870         {
871                 if(!background) {
872                         PixelMem pmem = pixel_mem_map[mem.device_pointer];
873
874                         cuda_push_context();
875
876                         /* for multi devices, this assumes the ineffecient method that we allocate
877                          * all pixels on the device even though we only render to a subset */
878                         size_t offset = sizeof(uint8_t)*4*y*w;
879
880                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pmem.cuPBO);
881                         glBindTexture(GL_TEXTURE_2D, pmem.cuTexId);
882                         glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, (void*)offset);
883                         glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
884                         
885                         glEnable(GL_TEXTURE_2D);
886                         
887                         if(transparent) {
888                                 glEnable(GL_BLEND);
889                                 glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
890                         }
891
892                         glColor3f(1.0f, 1.0f, 1.0f);
893
894                         glPushMatrix();
895                         glTranslatef(0.0f, (float)dy, 0.0f);
896                                 
897                         glBegin(GL_QUADS);
898                         
899                         glTexCoord2f(0.0f, 0.0f);
900                         glVertex2f(0.0f, 0.0f);
901                         glTexCoord2f((float)w/(float)pmem.w, 0.0f);
902                         glVertex2f((float)width, 0.0f);
903                         glTexCoord2f((float)w/(float)pmem.w, (float)h/(float)pmem.h);
904                         glVertex2f((float)width, (float)height);
905                         glTexCoord2f(0.0f, (float)h/(float)pmem.h);
906                         glVertex2f(0.0f, (float)height);
907
908                         glEnd();
909
910                         glPopMatrix();
911
912                         if(transparent)
913                                 glDisable(GL_BLEND);
914                         
915                         glBindTexture(GL_TEXTURE_2D, 0);
916                         glDisable(GL_TEXTURE_2D);
917
918                         cuda_pop_context();
919
920                         return;
921                 }
922
923                 Device::draw_pixels(mem, y, w, h, dy, width, height, transparent);
924         }
925
926         void thread_run(DeviceTask *task)
927         {
928                 if(task->type == DeviceTask::PATH_TRACE) {
929                         RenderTile tile;
930                         
931                         /* keep rendering tiles until done */
932                         while(task->acquire_tile(this, tile)) {
933                                 int start_sample = tile.start_sample;
934                                 int end_sample = tile.start_sample + tile.num_samples;
935
936                                 for(int sample = start_sample; sample < end_sample; sample++) {
937                                         if (task->get_cancel()) {
938                                                 if(task->need_finish_queue == false)
939                                                         break;
940                                         }
941
942                                         path_trace(tile, sample);
943
944                                         tile.sample = sample + 1;
945
946                                         task->update_progress(tile);
947                                 }
948
949                                 task->release_tile(tile);
950                         }
951                 }
952                 else if(task->type == DeviceTask::SHADER) {
953                         shader(*task);
954
955                         cuda_push_context();
956                         cuda_assert(cuCtxSynchronize())
957                         cuda_pop_context();
958                 }
959         }
960
961         class CUDADeviceTask : public DeviceTask {
962         public:
963                 CUDADeviceTask(CUDADevice *device, DeviceTask& task)
964                 : DeviceTask(task)
965                 {
966                         run = function_bind(&CUDADevice::thread_run, device, this);
967                 }
968         };
969
970         void task_add(DeviceTask& task)
971         {
972                 if(task.type == DeviceTask::TONEMAP) {
973                         /* must be done in main thread due to opengl access */
974                         tonemap(task, task.buffer, task.rgba);
975
976                         cuda_push_context();
977                         cuda_assert(cuCtxSynchronize())
978                         cuda_pop_context();
979                 }
980                 else {
981                         task_pool.push(new CUDADeviceTask(this, task));
982                 }
983         }
984
985         void task_wait()
986         {
987                 task_pool.wait_work();
988         }
989
990         void task_cancel()
991         {
992                 task_pool.cancel();
993         }
994 };
995
996 Device *device_cuda_create(DeviceInfo& info, Stats &stats, bool background)
997 {
998         return new CUDADevice(info, stats, background);
999 }
1000
1001 void device_cuda_info(vector<DeviceInfo>& devices)
1002 {
1003         CUresult result;
1004         int count = 0;
1005
1006         result = cuInit(0);
1007         if(result != CUDA_SUCCESS) {
1008                 if(result != CUDA_ERROR_NO_DEVICE)
1009                         fprintf(stderr, "CUDA cuInit: %s\n", CUDADevice::cuda_error_string(result));
1010                 return;
1011         }
1012
1013         result = cuDeviceGetCount(&count);
1014         if(result != CUDA_SUCCESS) {
1015                 fprintf(stderr, "CUDA cuDeviceGetCount: %s\n", CUDADevice::cuda_error_string(result));
1016                 return;
1017         }
1018         
1019         vector<DeviceInfo> display_devices;
1020         
1021         for(int num = 0; num < count; num++) {
1022                 char name[256];
1023                 int attr;
1024                 
1025                 if(cuDeviceGetName(name, 256, num) != CUDA_SUCCESS)
1026                         continue;
1027
1028                 DeviceInfo info;
1029
1030                 info.type = DEVICE_CUDA;
1031                 info.description = string(name);
1032                 info.id = string_printf("CUDA_%d", num);
1033                 info.num = num;
1034
1035                 int major, minor;
1036                 cuDeviceComputeCapability(&major, &minor, num);
1037                 info.advanced_shading = (major >= 2);
1038                 info.extended_images = (major >= 3);
1039                 info.pack_images = false;
1040
1041                 /* if device has a kernel timeout, assume it is used for display */
1042                 if(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, num) == CUDA_SUCCESS && attr == 1) {
1043                         info.display_device = true;
1044                         display_devices.push_back(info);
1045                 }
1046                 else
1047                         devices.push_back(info);
1048         }
1049
1050         if(!display_devices.empty())
1051                 devices.insert(devices.end(), display_devices.begin(), display_devices.end());
1052 }
1053
1054 CCL_NAMESPACE_END
1055