Cycles: Improve denoising speed on GPUs with small tile sizes
[blender.git] / intern / cycles / kernel / kernels / cuda / kernel_split.cu
1 /*
2  * Copyright 2011-2016 Blender Foundation
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 /* CUDA split kernel entry points */
18
19 #ifdef __CUDA_ARCH__
20
21 #define __SPLIT_KERNEL__
22
23 #include "kernel/kernel_compat_cuda.h"
24 #include "kernel_config.h"
25
26 #include "kernel/split/kernel_split_common.h"
27 #include "kernel/split/kernel_data_init.h"
28 #include "kernel/split/kernel_path_init.h"
29 #include "kernel/split/kernel_scene_intersect.h"
30 #include "kernel/split/kernel_lamp_emission.h"
31 #include "kernel/split/kernel_do_volume.h"
32 #include "kernel/split/kernel_queue_enqueue.h"
33 #include "kernel/split/kernel_indirect_background.h"
34 #include "kernel/split/kernel_shader_setup.h"
35 #include "kernel/split/kernel_shader_sort.h"
36 #include "kernel/split/kernel_shader_eval.h"
37 #include "kernel/split/kernel_holdout_emission_blurring_pathtermination_ao.h"
38 #include "kernel/split/kernel_subsurface_scatter.h"
39 #include "kernel/split/kernel_direct_lighting.h"
40 #include "kernel/split/kernel_shadow_blocked_ao.h"
41 #include "kernel/split/kernel_shadow_blocked_dl.h"
42 #include "kernel/split/kernel_enqueue_inactive.h"
43 #include "kernel/split/kernel_next_iteration_setup.h"
44 #include "kernel/split/kernel_indirect_subsurface.h"
45 #include "kernel/split/kernel_buffer_update.h"
46
47 #include "kernel/kernel_film.h"
48
49 /* kernels */
50 extern "C" __global__ void
51 CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_MAX_REGISTERS)
52 kernel_cuda_state_buffer_size(uint num_threads, uint64_t *size)
53 {
54         *size = split_data_buffer_size(NULL, num_threads);
55 }
56
57 extern "C" __global__ void
58 CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_MAX_REGISTERS)
59 kernel_cuda_path_trace_data_init(
60         ccl_global void *split_data_buffer,
61         int num_elements,
62         ccl_global char *ray_state,
63         int start_sample,
64         int end_sample,
65         int sx, int sy, int sw, int sh, int offset, int stride,
66         ccl_global int *Queue_index,
67         int queuesize,
68         ccl_global char *use_queues_flag,
69         ccl_global unsigned int *work_pool_wgs,
70         unsigned int num_samples,
71         ccl_global float *buffer)
72 {
73         kernel_data_init(NULL,
74                          NULL,
75                          split_data_buffer,
76                          num_elements,
77                          ray_state,
78                          start_sample,
79                          end_sample,
80                          sx, sy, sw, sh, offset, stride,
81                          Queue_index,
82                          queuesize,
83                          use_queues_flag,
84                          work_pool_wgs,
85                          num_samples,
86                          buffer);
87 }
88
89 #define DEFINE_SPLIT_KERNEL_FUNCTION(name) \
90         extern "C" __global__ void \
91         CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_SPLIT_MAX_REGISTERS) \
92         kernel_cuda_##name() \
93         { \
94                 kernel_##name(NULL); \
95         }
96
97 #define DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(name, type) \
98         extern "C" __global__ void \
99         CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_SPLIT_MAX_REGISTERS) \
100         kernel_cuda_##name() \
101         { \
102                 ccl_local type locals; \
103                 kernel_##name(NULL, &locals); \
104         }
105
106 DEFINE_SPLIT_KERNEL_FUNCTION(path_init)
107 DEFINE_SPLIT_KERNEL_FUNCTION(scene_intersect)
108 DEFINE_SPLIT_KERNEL_FUNCTION(lamp_emission)
109 DEFINE_SPLIT_KERNEL_FUNCTION(do_volume)
110 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(queue_enqueue, QueueEnqueueLocals)
111 DEFINE_SPLIT_KERNEL_FUNCTION(indirect_background)
112 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(shader_setup, uint)
113 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(shader_sort, ShaderSortLocals)
114 DEFINE_SPLIT_KERNEL_FUNCTION(shader_eval)
115 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(holdout_emission_blurring_pathtermination_ao, BackgroundAOLocals)
116 DEFINE_SPLIT_KERNEL_FUNCTION(subsurface_scatter)
117 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(direct_lighting, uint)
118 DEFINE_SPLIT_KERNEL_FUNCTION(shadow_blocked_ao)
119 DEFINE_SPLIT_KERNEL_FUNCTION(shadow_blocked_dl)
120 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(enqueue_inactive, uint)
121 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(next_iteration_setup, uint)
122 DEFINE_SPLIT_KERNEL_FUNCTION(indirect_subsurface)
123 DEFINE_SPLIT_KERNEL_FUNCTION_LOCALS(buffer_update, uint)
124
125 extern "C" __global__ void
126 CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_MAX_REGISTERS)
127 kernel_cuda_convert_to_byte(uchar4 *rgba, float *buffer, float sample_scale, int sx, int sy, int sw, int sh, int offset, int stride)
128 {
129         int x = sx + blockDim.x*blockIdx.x + threadIdx.x;
130         int y = sy + blockDim.y*blockIdx.y + threadIdx.y;
131
132         if(x < sx + sw && y < sy + sh)
133                 kernel_film_convert_to_byte(NULL, rgba, buffer, sample_scale, x, y, offset, stride);
134 }
135
136 extern "C" __global__ void
137 CUDA_LAUNCH_BOUNDS(CUDA_THREADS_BLOCK_WIDTH, CUDA_KERNEL_MAX_REGISTERS)
138 kernel_cuda_convert_to_half_float(uchar4 *rgba, float *buffer, float sample_scale, int sx, int sy, int sw, int sh, int offset, int stride)
139 {
140         int x = sx + blockDim.x*blockIdx.x + threadIdx.x;
141         int y = sy + blockDim.y*blockIdx.y + threadIdx.y;
142
143         if(x < sx + sw && y < sy + sh)
144                 kernel_film_convert_to_half_float(NULL, rgba, buffer, sample_scale, x, y, offset, stride);
145 }
146
147 #endif
148