intern/cycles/device/device_cpu.cpp

   1 /*
   2  * Copyright 2011-2013 Blender Foundation
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <stdlib.h>
  18 #include <string.h>
  19
  20 /* So ImathMath is included before our kernel_cpu_compat. */
  21 #ifdef WITH_OSL
  22 /* So no context pollution happens from indirectly included windows.h */
  23 #  include "util/util_windows.h"
  24 #  include <OSL/oslexec.h>
  25 #endif
  26
  27 #include "device/device.h"
  28 #include "device/device_denoising.h"
  29 #include "device/device_intern.h"
  30 #include "device/device_split_kernel.h"
  31
  32 #include "kernel/kernel.h"
  33 #include "kernel/kernel_compat_cpu.h"
  34 #include "kernel/kernel_types.h"
  35 #include "kernel/split/kernel_split_data.h"
  36 #include "kernel/kernel_globals.h"
  37
  38 #include "kernel/filter/filter.h"
  39
  40 #include "kernel/osl/osl_shader.h"
  41 #include "kernel/osl/osl_globals.h"
  42
  43 #include "render/buffers.h"
  44
  45 #include "util/util_debug.h"
  46 #include "util/util_foreach.h"
  47 #include "util/util_function.h"
  48 #include "util/util_logging.h"
  49 #include "util/util_map.h"
  50 #include "util/util_opengl.h"
  51 #include "util/util_progress.h"
  52 #include "util/util_system.h"
  53 #include "util/util_thread.h"
  54
  55 CCL_NAMESPACE_BEGIN
  56
  57 class CPUDevice;
  58
  59 /* Has to be outside of the class to be shared across template instantiations. */
  60 static const char *logged_architecture = "";
  61
  62 template<typename F>
  63 class KernelFunctions {
  64 public:
  65         KernelFunctions()
  66         {
  67                 kernel = (F)NULL;
  68         }
  69
  70         KernelFunctions(F kernel_default,
  71                         F kernel_sse2,
  72                         F kernel_sse3,
  73                         F kernel_sse41,
  74                         F kernel_avx,
  75                         F kernel_avx2)
  76         {
  77                 const char *architecture_name = "default";
  78                 kernel = kernel_default;
  79
  80                 /* Silence potential warnings about unused variables
  81                  * when compiling without some architectures. */
  82                 (void)kernel_sse2;
  83                 (void)kernel_sse3;
  84                 (void)kernel_sse41;
  85                 (void)kernel_avx;
  86                 (void)kernel_avx2;
  87 #ifdef WITH_CYCLES_OPTIMIZED_KERNEL_AVX2
  88                 if(system_cpu_support_avx2()) {
  89                         architecture_name = "AVX2";
  90                         kernel = kernel_avx2;
  91                 }
  92                 else
  93 #endif
  94 #ifdef WITH_CYCLES_OPTIMIZED_KERNEL_AVX
  95                 if(system_cpu_support_avx()) {
  96                         architecture_name = "AVX";
  97                         kernel = kernel_avx;
  98                 }
  99                 else
 100 #endif
 101 #ifdef WITH_CYCLES_OPTIMIZED_KERNEL_SSE41
 102                 if(system_cpu_support_sse41()) {
 103                         architecture_name = "SSE4.1";
 104                         kernel = kernel_sse41;
 105                 }
 106                 else
 107 #endif
 108 #ifdef WITH_CYCLES_OPTIMIZED_KERNEL_SSE3
 109                 if(system_cpu_support_sse3()) {
 110                         architecture_name = "SSE3";
 111                         kernel = kernel_sse3;
 112                 }
 113                 else
 114 #endif
 115 #ifdef WITH_CYCLES_OPTIMIZED_KERNEL_SSE2
 116                 if(system_cpu_support_sse2()) {
 117                         architecture_name = "SSE2";
 118                         kernel = kernel_sse2;
 119                 }
 120 #endif
 121
 122                 if(strstr(architecture_name, logged_architecture) != 0) {
 123                         VLOG(1) << "Will be using " << architecture_name << " kernels.";
 124                         logged_architecture = architecture_name;
 125                 }
 126         }
 127
 128         inline F operator()() const {
 129                 assert(kernel);
 130                 return kernel;
 131         }
 132 protected:
 133         F kernel;
 134 };
 135
 136 class CPUSplitKernel : public DeviceSplitKernel {
 137         CPUDevice *device;
 138 public:
 139         explicit CPUSplitKernel(CPUDevice *device);
 140
 141         virtual bool enqueue_split_kernel_data_init(const KernelDimensions& dim,
 142                                                     RenderTile& rtile,
 143                                                     int num_global_elements,
 144                                                     device_memory& kernel_globals,
 145                                                     device_memory& kernel_data_,
 146                                                     device_memory& split_data,
 147                                                     device_memory& ray_state,
 148                                                     device_memory& queue_index,
 149                                                     device_memory& use_queues_flag,
 150                                                     device_memory& work_pool_wgs);
 151
 152         virtual SplitKernelFunction* get_split_kernel_function(string kernel_name, const DeviceRequestedFeatures&);
 153         virtual int2 split_kernel_local_size();
 154         virtual int2 split_kernel_global_size(device_memory& kg, device_memory& data, DeviceTask *task);
 155         virtual uint64_t state_buffer_size(device_memory& kg, device_memory& data, size_t num_threads);
 156 };
 157
 158 class CPUDevice : public Device
 159 {
 160 public:
 161         TaskPool task_pool;
 162         KernelGlobals kernel_globals;
 163
 164 #ifdef WITH_OSL
 165         OSLGlobals osl_globals;
 166 #endif
 167
 168         bool use_split_kernel;
 169
 170         DeviceRequestedFeatures requested_features;
 171
 172         KernelFunctions<void(*)(KernelGlobals *, float *, unsigned int *, int, int, int, int, int)>   path_trace_kernel;
 173         KernelFunctions<void(*)(KernelGlobals *, uchar4 *, float *, float, int, int, int, int)>       convert_to_half_float_kernel;
 174         KernelFunctions<void(*)(KernelGlobals *, uchar4 *, float *, float, int, int, int, int)>       convert_to_byte_kernel;
 175         KernelFunctions<void(*)(KernelGlobals *, uint4 *, float4 *, float*, int, int, int, int, int)> shader_kernel;
 176
 177         KernelFunctions<void(*)(int, TilesInfo*, int, int, float*, float*, float*, float*, float*, int*, int, int, bool)> filter_divide_shadow_kernel;
 178         KernelFunctions<void(*)(int, TilesInfo*, int, int, int, int, float*, float*, int*, int, int, bool)>               filter_get_feature_kernel;
 179         KernelFunctions<void(*)(int, int, float*, float*, float*, float*, int*, int)>                                     filter_combine_halves_kernel;
 180
 181         KernelFunctions<void(*)(int, int, float*, float*, float*, int*, int, int, float, float)> filter_nlm_calc_difference_kernel;
 182         KernelFunctions<void(*)(float*, float*, int*, int, int)>                                 filter_nlm_blur_kernel;
 183         KernelFunctions<void(*)(float*, float*, int*, int, int)>                                 filter_nlm_calc_weight_kernel;
 184         KernelFunctions<void(*)(int, int, float*, float*, float*, float*, int*, int, int)>       filter_nlm_update_output_kernel;
 185         KernelFunctions<void(*)(float*, float*, int*, int)>                                      filter_nlm_normalize_kernel;
 186
 187         KernelFunctions<void(*)(float*, int, int, int, float*, int*, int*, int, int, float)>                                              filter_construct_transform_kernel;
 188         KernelFunctions<void(*)(int, int, float*, float*, float*, float*, float*, int*, float*, float3*, int*, int*, int, int, int, int)> filter_nlm_construct_gramian_kernel;
 189         KernelFunctions<void(*)(int, int, int, int, int, float*, int*, float*, float3*, int*, int)>                                       filter_finalize_kernel;
 190
 191         KernelFunctions<void(*)(KernelGlobals *, ccl_constant KernelData*, ccl_global void*, int, ccl_global char*,
 192                                ccl_global uint*, int, int, int, int, int, int, int, int, ccl_global int*, int,
 193                                ccl_global char*, ccl_global unsigned int*, unsigned int, ccl_global float*)>        data_init_kernel;
 194         unordered_map<string, KernelFunctions<void(*)(KernelGlobals*, KernelData*)> > split_kernels;
 195
 196 #define KERNEL_FUNCTIONS(name) \
 197               KERNEL_NAME_EVAL(cpu, name), \
 198               KERNEL_NAME_EVAL(cpu_sse2, name), \
 199               KERNEL_NAME_EVAL(cpu_sse3, name), \
 200               KERNEL_NAME_EVAL(cpu_sse41, name), \
 201               KERNEL_NAME_EVAL(cpu_avx, name), \
 202               KERNEL_NAME_EVAL(cpu_avx2, name)
 203
 204         CPUDevice(DeviceInfo& info, Stats &stats, bool background)
 205         : Device(info, stats, background),
 206 #define REGISTER_KERNEL(name) name ## _kernel(KERNEL_FUNCTIONS(name))
 207           REGISTER_KERNEL(path_trace),
 208           REGISTER_KERNEL(convert_to_half_float),
 209           REGISTER_KERNEL(convert_to_byte),
 210           REGISTER_KERNEL(shader),
 211           REGISTER_KERNEL(filter_divide_shadow),
 212           REGISTER_KERNEL(filter_get_feature),
 213           REGISTER_KERNEL(filter_combine_halves),
 214           REGISTER_KERNEL(filter_nlm_calc_difference),
 215           REGISTER_KERNEL(filter_nlm_blur),
 216           REGISTER_KERNEL(filter_nlm_calc_weight),
 217           REGISTER_KERNEL(filter_nlm_update_output),
 218           REGISTER_KERNEL(filter_nlm_normalize),
 219           REGISTER_KERNEL(filter_construct_transform),
 220           REGISTER_KERNEL(filter_nlm_construct_gramian),
 221           REGISTER_KERNEL(filter_finalize),
 222           REGISTER_KERNEL(data_init)
 223 #undef REGISTER_KERNEL
 224         {
 225
 226 #ifdef WITH_OSL
 227                 kernel_globals.osl = &osl_globals;
 228 #endif
 229                 use_split_kernel = DebugFlags().cpu.split_kernel;
 230                 if(use_split_kernel) {
 231                         VLOG(1) << "Will be using split kernel.";
 232                 }
 233
 234 #define REGISTER_SPLIT_KERNEL(name) split_kernels[#name] = KernelFunctions<void(*)(KernelGlobals*, KernelData*)>(KERNEL_FUNCTIONS(name))
 235                 REGISTER_SPLIT_KERNEL(path_init);
 236                 REGISTER_SPLIT_KERNEL(scene_intersect);
 237                 REGISTER_SPLIT_KERNEL(lamp_emission);
 238                 REGISTER_SPLIT_KERNEL(do_volume);
 239                 REGISTER_SPLIT_KERNEL(queue_enqueue);
 240                 REGISTER_SPLIT_KERNEL(indirect_background);
 241                 REGISTER_SPLIT_KERNEL(shader_setup);
 242                 REGISTER_SPLIT_KERNEL(shader_sort);
 243                 REGISTER_SPLIT_KERNEL(shader_eval);
 244                 REGISTER_SPLIT_KERNEL(holdout_emission_blurring_pathtermination_ao);
 245                 REGISTER_SPLIT_KERNEL(subsurface_scatter);
 246                 REGISTER_SPLIT_KERNEL(direct_lighting);
 247                 REGISTER_SPLIT_KERNEL(shadow_blocked_ao);
 248                 REGISTER_SPLIT_KERNEL(shadow_blocked_dl);
 249                 REGISTER_SPLIT_KERNEL(next_iteration_setup);
 250                 REGISTER_SPLIT_KERNEL(indirect_subsurface);
 251                 REGISTER_SPLIT_KERNEL(buffer_update);
 252 #undef REGISTER_SPLIT_KERNEL
 253 #undef KERNEL_FUNCTIONS
 254         }
 255
 256         ~CPUDevice()
 257         {
 258                 task_pool.stop();
 259         }
 260
 261         virtual bool show_samples() const
 262         {
 263                 return (TaskScheduler::num_threads() == 1);
 264         }
 265
 266         void mem_alloc(const char *name, device_memory& mem, MemoryType /*type*/)
 267         {
 268                 if(name) {
 269                         VLOG(1) << "Buffer allocate: " << name << ", "
 270                                 << string_human_readable_number(mem.memory_size()) << " bytes. ("
 271                                 << string_human_readable_size(mem.memory_size()) << ")";
 272                 }
 273
 274                 mem.device_pointer = mem.data_pointer;
 275
 276                 if(!mem.device_pointer) {
 277                         mem.device_pointer = (device_ptr)malloc(mem.memory_size());
 278                 }
 279
 280                 mem.device_size = mem.memory_size();
 281                 stats.mem_alloc(mem.device_size);
 282         }
 283
 284         void mem_copy_to(device_memory& /*mem*/)
 285         {
 286                 /* no-op */
 287         }
 288
 289         void mem_copy_from(device_memory& /*mem*/,
 290                            int /*y*/, int /*w*/, int /*h*/,
 291                            int /*elem*/)
 292         {
 293                 /* no-op */
 294         }
 295
 296         void mem_zero(device_memory& mem)
 297         {
 298                 memset((void*)mem.device_pointer, 0, mem.memory_size());
 299         }
 300
 301         void mem_free(device_memory& mem)
 302         {
 303                 if(mem.device_pointer) {
 304                         if(!mem.data_pointer) {
 305                                 free((void*)mem.device_pointer);
 306                         }
 307                         mem.device_pointer = 0;
 308                         stats.mem_free(mem.device_size);
 309                         mem.device_size = 0;
 310                 }
 311         }
 312
 313         virtual device_ptr mem_alloc_sub_ptr(device_memory& mem, int offset, int /*size*/, MemoryType /*type*/)
 314         {
 315                 return (device_ptr) (((char*) mem.device_pointer) + mem.memory_elements_size(offset));
 316         }
 317
 318         void const_copy_to(const char *name, void *host, size_t size)
 319         {
 320                 kernel_const_copy(&kernel_globals, name, host, size);
 321         }
 322
 323         void tex_alloc(const char *name,
 324                        device_memory& mem,
 325                        InterpolationType interpolation,
 326                        ExtensionType extension)
 327         {
 328                 VLOG(1) << "Texture allocate: " << name << ", "
 329                         << string_human_readable_number(mem.memory_size()) << " bytes. ("
 330                         << string_human_readable_size(mem.memory_size()) << ")";
 331                 kernel_tex_copy(&kernel_globals,
 332                                 name,
 333                                 mem.data_pointer,
 334                                 mem.data_width,
 335                                 mem.data_height,
 336                                 mem.data_depth,
 337                                 interpolation,
 338                                 extension);
 339                 mem.device_pointer = mem.data_pointer;
 340                 mem.device_size = mem.memory_size();
 341                 stats.mem_alloc(mem.device_size);
 342         }
 343
 344         void tex_free(device_memory& mem)
 345         {
 346                 if(mem.device_pointer) {
 347                         mem.device_pointer = 0;
 348                         stats.mem_free(mem.device_size);
 349                         mem.device_size = 0;
 350                 }
 351         }
 352
 353         void *osl_memory()
 354         {
 355 #ifdef WITH_OSL
 356                 return &osl_globals;
 357 #else
 358                 return NULL;
 359 #endif
 360         }
 361
 362         void thread_run(DeviceTask *task)
 363         {
 364                 if(task->type == DeviceTask::RENDER) {
 365                         thread_render(*task);
 366                 }
 367                 else if(task->type == DeviceTask::FILM_CONVERT)
 368                         thread_film_convert(*task);
 369                 else if(task->type == DeviceTask::SHADER)
 370                         thread_shader(*task);
 371         }
 372
 373         class CPUDeviceTask : public DeviceTask {
 374         public:
 375                 CPUDeviceTask(CPUDevice *device, DeviceTask& task)
 376                 : DeviceTask(task)
 377                 {
 378                         run = function_bind(&CPUDevice::thread_run, device, this);
 379                 }
 380         };
 381
 382         bool denoising_set_tiles(device_ptr *buffers, DenoisingTask *task)
 383         {
 384                 mem_alloc("Denoising Tile Info", task->tiles_mem, MEM_READ_ONLY);
 385
 386                 TilesInfo *tiles = (TilesInfo*) task->tiles_mem.data_pointer;
 387                 for(int i = 0; i < 9; i++) {
 388                         tiles->buffers[i] = buffers[i];
 389                 }
 390
 391                 return true;
 392         }
 393
 394         bool denoising_non_local_means(device_ptr image_ptr, device_ptr guide_ptr, device_ptr variance_ptr, device_ptr out_ptr,
 395                                        DenoisingTask *task)
 396         {
 397                 int4 rect = task->rect;
 398                 int   r   = task->nlm_state.r;
 399                 int   f   = task->nlm_state.f;
 400                 float a   = task->nlm_state.a;
 401                 float k_2 = task->nlm_state.k_2;
 402
 403                 int w = align_up(rect.z-rect.x, 4);
 404                 int h = rect.w-rect.y;
 405
 406                 float *blurDifference = (float*) task->nlm_state.temporary_1_ptr;
 407                 float *difference     = (float*) task->nlm_state.temporary_2_ptr;
 408                 float *weightAccum    = (float*) task->nlm_state.temporary_3_ptr;
 409
 410                 memset(weightAccum, 0, sizeof(float)*w*h);
 411                 memset((float*) out_ptr, 0, sizeof(float)*w*h);
 412
 413                 for(int i = 0; i < (2*r+1)*(2*r+1); i++) {
 414                         int dy = i / (2*r+1) - r;
 415                         int dx = i % (2*r+1) - r;
 416
 417                         int local_rect[4] = {max(0, -dx), max(0, -dy), rect.z-rect.x - max(0, dx), rect.w-rect.y - max(0, dy)};
 418                         filter_nlm_calc_difference_kernel()(dx, dy,
 419                                                             (float*) guide_ptr,
 420                                                             (float*) variance_ptr,
 421                                                             difference,
 422                                                             local_rect,
 423                                                             w, 0,
 424                                                             a, k_2);
 425
 426                         filter_nlm_blur_kernel()       (difference, blurDifference, local_rect, w, f);
 427                         filter_nlm_calc_weight_kernel()(blurDifference, difference, local_rect, w, f);
 428                         filter_nlm_blur_kernel()       (difference, blurDifference, local_rect, w, f);
 429
 430                         filter_nlm_update_output_kernel()(dx, dy,
 431                                                           blurDifference,
 432                                                           (float*) image_ptr,
 433                                                           (float*) out_ptr,
 434                                                           weightAccum,
 435                                                           local_rect,
 436                                                           w, f);
 437                 }
 438
 439                 int local_rect[4] = {0, 0, rect.z-rect.x, rect.w-rect.y};
 440                 filter_nlm_normalize_kernel()((float*) out_ptr, weightAccum, local_rect, w);
 441
 442                 return true;
 443         }
 444
 445         bool denoising_construct_transform(DenoisingTask *task)
 446         {
 447                 for(int y = 0; y < task->filter_area.w; y++) {
 448                         for(int x = 0; x < task->filter_area.z; x++) {
 449                                 filter_construct_transform_kernel()((float*) task->buffer.mem.device_pointer,
 450                                                                     x + task->filter_area.x,
 451                                                                     y + task->filter_area.y,
 452                                                                     y*task->filter_area.z + x,
 453                                                                     (float*) task->storage.transform.device_pointer,
 454                                                                     (int*)   task->storage.rank.device_pointer,
 455                                                                     &task->rect.x,
 456                                                                     task->buffer.pass_stride,
 457                                                                     task->radius,
 458                                                                     task->pca_threshold);
 459                         }
 460                 }
 461                 return true;
 462         }
 463
 464         bool denoising_reconstruct(device_ptr color_ptr,
 465                                    device_ptr color_variance_ptr,
 466                                    device_ptr guide_ptr,
 467                                    device_ptr guide_variance_ptr,
 468                                    device_ptr output_ptr,
 469                                    DenoisingTask *task)
 470         {
 471                 mem_zero(task->storage.XtWX);
 472                 mem_zero(task->storage.XtWY);
 473
 474                 float *difference     = (float*) task->reconstruction_state.temporary_1_ptr;
 475                 float *blurDifference = (float*) task->reconstruction_state.temporary_2_ptr;
 476
 477                 int r = task->radius;
 478                 for(int i = 0; i < (2*r+1)*(2*r+1); i++) {
 479                         int dy = i / (2*r+1) - r;
 480                         int dx = i % (2*r+1) - r;
 481
 482                         int local_rect[4] = {max(0, -dx), max(0, -dy),
 483                                              task->reconstruction_state.source_w - max(0, dx),
 484                                              task->reconstruction_state.source_h - max(0, dy)};
 485                         filter_nlm_calc_difference_kernel()(dx, dy,
 486                                                             (float*) guide_ptr,
 487                                                             (float*) guide_variance_ptr,
 488                                                             difference,
 489                                                             local_rect,
 490                                                             task->buffer.w,
 491                                                             task->buffer.pass_stride,
 492                                                             1.0f,
 493                                                             task->nlm_k_2);
 494                         filter_nlm_blur_kernel()(difference, blurDifference, local_rect, task->buffer.w, 4);
 495                         filter_nlm_calc_weight_kernel()(blurDifference, difference, local_rect, task->buffer.w, 4);
 496                         filter_nlm_blur_kernel()(difference, blurDifference, local_rect, task->buffer.w, 4);
 497                         filter_nlm_construct_gramian_kernel()(dx, dy,
 498                                                               blurDifference,
 499                                                               (float*)  task->buffer.mem.device_pointer,
 500                                                               (float*)  color_ptr,
 501                                                               (float*)  color_variance_ptr,
 502                                                               (float*)  task->storage.transform.device_pointer,
 503                                                               (int*)    task->storage.rank.device_pointer,
 504                                                               (float*)  task->storage.XtWX.device_pointer,
 505                                                               (float3*) task->storage.XtWY.device_pointer,
 506                                                               local_rect,
 507                                                               &task->reconstruction_state.filter_rect.x,
 508                                                               task->buffer.w,
 509                                                               task->buffer.h,
 510                                                               4,
 511                                                               task->buffer.pass_stride);
 512                 }
 513                 for(int y = 0; y < task->filter_area.w; y++) {
 514                         for(int x = 0; x < task->filter_area.z; x++) {
 515                                 filter_finalize_kernel()(x,
 516                                                          y,
 517                                                          y*task->filter_area.z + x,
 518                                                          task->buffer.w,
 519                                                          task->buffer.h,
 520                                                          (float*)  output_ptr,
 521                                                          (int*)    task->storage.rank.device_pointer,
 522                                                          (float*)  task->storage.XtWX.device_pointer,
 523                                                          (float3*) task->storage.XtWY.device_pointer,
 524                                                          &task->reconstruction_state.buffer_params.x,
 525                                                          task->render_buffer.samples);
 526                         }
 527                 }
 528                 return true;
 529         }
 530
 531         bool denoising_combine_halves(device_ptr a_ptr, device_ptr b_ptr,
 532                                       device_ptr mean_ptr, device_ptr variance_ptr,
 533                                       int r, int4 rect, DenoisingTask *task)
 534         {
 535                 (void) task;
 536                 for(int y = rect.y; y < rect.w; y++) {
 537                         for(int x = rect.x; x < rect.z; x++) {
 538                                 filter_combine_halves_kernel()(x, y,
 539                                                                (float*) mean_ptr,
 540                                                                (float*) variance_ptr,
 541                                                                (float*) a_ptr,
 542                                                                (float*) b_ptr,
 543                                                                &rect.x,
 544                                                                r);
 545                         }
 546                 }
 547                 return true;
 548         }
 549
 550         bool denoising_divide_shadow(device_ptr a_ptr, device_ptr b_ptr,
 551                                      device_ptr sample_variance_ptr, device_ptr sv_variance_ptr,
 552                                      device_ptr buffer_variance_ptr, DenoisingTask *task)
 553         {
 554                 for(int y = task->rect.y; y < task->rect.w; y++) {
 555                         for(int x = task->rect.x; x < task->rect.z; x++) {
 556                                 filter_divide_shadow_kernel()(task->render_buffer.samples,
 557                                                               task->tiles,
 558                                                               x, y,
 559                                                               (float*) a_ptr,
 560                                                               (float*) b_ptr,
 561                                                               (float*) sample_variance_ptr,
 562                                                               (float*) sv_variance_ptr,
 563                                                               (float*) buffer_variance_ptr,
 564                                                               &task->rect.x,
 565                                                               task->render_buffer.pass_stride,
 566                                                               task->render_buffer.denoising_data_offset,
 567                                                               use_split_kernel);
 568                         }
 569                 }
 570                 return true;
 571         }
 572
 573         bool denoising_get_feature(int mean_offset,
 574                                    int variance_offset,
 575                                    device_ptr mean_ptr,
 576                                    device_ptr variance_ptr,
 577                                    DenoisingTask *task)
 578         {
 579                 for(int y = task->rect.y; y < task->rect.w; y++) {
 580                         for(int x = task->rect.x; x < task->rect.z; x++) {
 581                                 filter_get_feature_kernel()(task->render_buffer.samples,
 582                                                             task->tiles,
 583                                                             mean_offset,
 584                                                             variance_offset,
 585                                                             x, y,
 586                                                             (float*) mean_ptr,
 587                                                             (float*) variance_ptr,
 588                                                             &task->rect.x,
 589                                                             task->render_buffer.pass_stride,
 590                                                             task->render_buffer.denoising_data_offset,
 591                                                             use_split_kernel);
 592                         }
 593                 }
 594                 return true;
 595         }
 596
 597         void path_trace(DeviceTask &task, RenderTile &tile, KernelGlobals *kg)
 598         {
 599                 float *render_buffer = (float*)tile.buffer;
 600                 uint *rng_state = (uint*)tile.rng_state;
 601                 int start_sample = tile.start_sample;
 602                 int end_sample = tile.start_sample + tile.num_samples;
 603
 604                 for(int sample = start_sample; sample < end_sample; sample++) {
 605                         if(task.get_cancel() || task_pool.canceled()) {
 606                                 if(task.need_finish_queue == false)
 607                                         break;
 608                         }
 609
 610                         for(int y = tile.y; y < tile.y + tile.h; y++) {
 611                                 for(int x = tile.x; x < tile.x + tile.w; x++) {
 612                                         path_trace_kernel()(kg, render_buffer, rng_state,
 613                                                             sample, x, y, tile.offset, tile.stride);
 614                                 }
 615                         }
 616
 617                         tile.sample = sample + 1;
 618
 619                         task.update_progress(&tile, tile.w*tile.h);
 620                 }
 621         }
 622
 623         void denoise(DeviceTask &task, RenderTile &tile)
 624         {
 625                 tile.sample = tile.start_sample + tile.num_samples;
 626
 627                 DenoisingTask denoising(this);
 628
 629                 denoising.functions.construct_transform = function_bind(&CPUDevice::denoising_construct_transform, this, &denoising);
 630                 denoising.functions.reconstruct = function_bind(&CPUDevice::denoising_reconstruct, this, _1, _2, _3, _4, _5, &denoising);
 631                 denoising.functions.divide_shadow = function_bind(&CPUDevice::denoising_divide_shadow, this, _1, _2, _3, _4, _5, &denoising);
 632                 denoising.functions.non_local_means = function_bind(&CPUDevice::denoising_non_local_means, this, _1, _2, _3, _4, &denoising);
 633                 denoising.functions.combine_halves = function_bind(&CPUDevice::denoising_combine_halves, this, _1, _2, _3, _4, _5, _6, &denoising);
 634                 denoising.functions.get_feature = function_bind(&CPUDevice::denoising_get_feature, this, _1, _2, _3, _4, &denoising);
 635                 denoising.functions.set_tiles = function_bind(&CPUDevice::denoising_set_tiles, this, _1, &denoising);
 636
 637                 denoising.filter_area = make_int4(tile.x, tile.y, tile.w, tile.h);
 638                 denoising.render_buffer.samples = tile.sample;
 639
 640                 RenderTile rtiles[9];
 641                 rtiles[4] = tile;
 642                 task.map_neighbor_tiles(rtiles, this);
 643                 denoising.tiles_from_rendertiles(rtiles);
 644
 645                 denoising.init_from_devicetask(task);
 646
 647                 denoising.run_denoising();
 648
 649                 task.unmap_neighbor_tiles(rtiles, this);
 650
 651                 task.update_progress(&tile, tile.w*tile.h);
 652         }
 653
 654         void thread_render(DeviceTask& task)
 655         {
 656                 if(task_pool.canceled()) {
 657                         if(task.need_finish_queue == false)
 658                                 return;
 659                 }
 660
 661                 /* allocate buffer for kernel globals */
 662                 device_only_memory<KernelGlobals> kgbuffer;
 663                 kgbuffer.resize(1);
 664                 mem_alloc("kernel_globals", kgbuffer, MEM_READ_WRITE);
 665
 666                 KernelGlobals *kg = new ((void*) kgbuffer.device_pointer) KernelGlobals(thread_kernel_globals_init());
 667
 668                 CPUSplitKernel *split_kernel = NULL;
 669                 if(use_split_kernel) {
 670                         split_kernel = new CPUSplitKernel(this);
 671                         requested_features.max_closure = MAX_CLOSURE;
 672                         if(!split_kernel->load_kernels(requested_features)) {
 673                                 thread_kernel_globals_free((KernelGlobals*)kgbuffer.device_pointer);
 674                                 mem_free(kgbuffer);
 675
 676                                 delete split_kernel;
 677                                 return;
 678                         }
 679                 }
 680
 681                 RenderTile tile;
 682                 while(task.acquire_tile(this, tile)) {
 683                         if(tile.task == RenderTile::PATH_TRACE) {
 684                                 if(use_split_kernel) {
 685                                         device_memory data;
 686                                         split_kernel->path_trace(&task, tile, kgbuffer, data);
 687                                 }
 688                                 else {
 689                                         path_trace(task, tile, kg);
 690                                 }
 691                         }
 692                         else if(tile.task == RenderTile::DENOISE) {
 693                                 denoise(task, tile);
 694                         }
 695
 696                         task.release_tile(tile);
 697
 698                         if(task_pool.canceled()) {
 699                                 if(task.need_finish_queue == false)
 700                                         break;
 701                         }
 702                 }
 703
 704                 thread_kernel_globals_free((KernelGlobals*)kgbuffer.device_pointer);
 705                 mem_free(kgbuffer);
 706                 delete split_kernel;
 707         }
 708
 709         void thread_film_convert(DeviceTask& task)
 710         {
 711                 float sample_scale = 1.0f/(task.sample + 1);
 712
 713                 if(task.rgba_half) {
 714                         for(int y = task.y; y < task.y + task.h; y++)
 715                                 for(int x = task.x; x < task.x + task.w; x++)
 716                                         convert_to_half_float_kernel()(&kernel_globals, (uchar4*)task.rgba_half, (float*)task.buffer,
 717                                                                        sample_scale, x, y, task.offset, task.stride);
 718                 }
 719                 else {
 720                         for(int y = task.y; y < task.y + task.h; y++)
 721                                 for(int x = task.x; x < task.x + task.w; x++)
 722                                         convert_to_byte_kernel()(&kernel_globals, (uchar4*)task.rgba_byte, (float*)task.buffer,
 723                                                                  sample_scale, x, y, task.offset, task.stride);
 724
 725                 }
 726         }
 727
 728         void thread_shader(DeviceTask& task)
 729         {
 730                 KernelGlobals kg = kernel_globals;
 731
 732 #ifdef WITH_OSL
 733                 OSLShader::thread_init(&kg, &kernel_globals, &osl_globals);
 734 #endif
 735                 for(int sample = 0; sample < task.num_samples; sample++) {
 736                         for(int x = task.shader_x; x < task.shader_x + task.shader_w; x++)
 737                                 shader_kernel()(&kg,
 738                                                 (uint4*)task.shader_input,
 739                                                 (float4*)task.shader_output,
 740                                                 (float*)task.shader_output_luma,
 741                                                 task.shader_eval_type,
 742                                                 task.shader_filter,
 743                                                 x,
 744                                                 task.offset,
 745                                                 sample);
 746
 747                         if(task.get_cancel() || task_pool.canceled())
 748                                 break;
 749
 750                         task.update_progress(NULL);
 751
 752                 }
 753
 754 #ifdef WITH_OSL
 755                 OSLShader::thread_free(&kg);
 756 #endif
 757         }
 758
 759         int get_split_task_count(DeviceTask& task)
 760         {
 761                 if(task.type == DeviceTask::SHADER)
 762                         return task.get_subtask_count(TaskScheduler::num_threads(), 256);
 763                 else
 764                         return task.get_subtask_count(TaskScheduler::num_threads());
 765         }
 766
 767         void task_add(DeviceTask& task)
 768         {
 769                 /* split task into smaller ones */
 770                 list<DeviceTask> tasks;
 771
 772                 if(task.type == DeviceTask::SHADER)
 773                         task.split(tasks, TaskScheduler::num_threads(), 256);
 774                 else
 775                         task.split(tasks, TaskScheduler::num_threads());
 776
 777                 foreach(DeviceTask& task, tasks)
 778                         task_pool.push(new CPUDeviceTask(this, task));
 779         }
 780
 781         void task_wait()
 782         {
 783                 task_pool.wait_work();
 784         }
 785
 786         void task_cancel()
 787         {
 788                 task_pool.cancel();
 789         }
 790
 791 protected:
 792         inline KernelGlobals thread_kernel_globals_init()
 793         {
 794                 KernelGlobals kg = kernel_globals;
 795                 kg.transparent_shadow_intersections = NULL;
 796                 const int decoupled_count = sizeof(kg.decoupled_volume_steps) /
 797                                             sizeof(*kg.decoupled_volume_steps);
 798                 for(int i = 0; i < decoupled_count; ++i) {
 799                         kg.decoupled_volume_steps[i] = NULL;
 800                 }
 801                 kg.decoupled_volume_steps_index = 0;
 802 #ifdef WITH_OSL
 803                 OSLShader::thread_init(&kg, &kernel_globals, &osl_globals);
 804 #endif
 805                 return kg;
 806         }
 807
 808         inline void thread_kernel_globals_free(KernelGlobals *kg)
 809         {
 810                 if(kg == NULL) {
 811                         return;
 812                 }
 813
 814                 if(kg->transparent_shadow_intersections != NULL) {
 815                         free(kg->transparent_shadow_intersections);
 816                 }
 817                 const int decoupled_count = sizeof(kg->decoupled_volume_steps) /
 818                                             sizeof(*kg->decoupled_volume_steps);
 819                 for(int i = 0; i < decoupled_count; ++i) {
 820                         if(kg->decoupled_volume_steps[i] != NULL) {
 821                                 free(kg->decoupled_volume_steps[i]);
 822                         }
 823                 }
 824 #ifdef WITH_OSL
 825                 OSLShader::thread_free(kg);
 826 #endif
 827         }
 828
 829         virtual bool load_kernels(DeviceRequestedFeatures& requested_features_) {
 830                 requested_features = requested_features_;
 831
 832                 return true;
 833         }
 834 };
 835
 836 /* split kernel */
 837
 838 class CPUSplitKernelFunction : public SplitKernelFunction {
 839 public:
 840         CPUDevice* device;
 841         void (*func)(KernelGlobals *kg, KernelData *data);
 842
 843         CPUSplitKernelFunction(CPUDevice* device) : device(device), func(NULL) {}
 844         ~CPUSplitKernelFunction() {}
 845
 846         virtual bool enqueue(const KernelDimensions& dim, device_memory& kernel_globals, device_memory& data)
 847         {
 848                 if(!func) {
 849                         return false;
 850                 }
 851
 852                 KernelGlobals *kg = (KernelGlobals*)kernel_globals.device_pointer;
 853                 kg->global_size = make_int2(dim.global_size[0], dim.global_size[1]);
 854
 855                 for(int y = 0; y < dim.global_size[1]; y++) {
 856                         for(int x = 0; x < dim.global_size[0]; x++) {
 857                                 kg->global_id = make_int2(x, y);
 858
 859                                 func(kg, (KernelData*)data.device_pointer);
 860                         }
 861                 }
 862
 863                 return true;
 864         }
 865 };
 866
 867 CPUSplitKernel::CPUSplitKernel(CPUDevice *device) : DeviceSplitKernel(device), device(device)
 868 {
 869 }
 870
 871 bool CPUSplitKernel::enqueue_split_kernel_data_init(const KernelDimensions& dim,
 872                                                     RenderTile& rtile,
 873                                                     int num_global_elements,
 874                                                     device_memory& kernel_globals,
 875                                                     device_memory& data,
 876                                                     device_memory& split_data,
 877                                                     device_memory& ray_state,
 878                                                     device_memory& queue_index,
 879                                                     device_memory& use_queues_flags,
 880                                                     device_memory& work_pool_wgs)
 881 {
 882         KernelGlobals *kg = (KernelGlobals*)kernel_globals.device_pointer;
 883         kg->global_size = make_int2(dim.global_size[0], dim.global_size[1]);
 884
 885         for(int y = 0; y < dim.global_size[1]; y++) {
 886                 for(int x = 0; x < dim.global_size[0]; x++) {
 887                         kg->global_id = make_int2(x, y);
 888
 889                         device->data_init_kernel()((KernelGlobals*)kernel_globals.device_pointer,
 890                                                    (KernelData*)data.device_pointer,
 891                                                    (void*)split_data.device_pointer,
 892                                                    num_global_elements,
 893                                                    (char*)ray_state.device_pointer,
 894                                                    (uint*)rtile.rng_state,
 895                                                    rtile.start_sample,
 896                                                    rtile.start_sample + rtile.num_samples,
 897                                                    rtile.x,
 898                                                    rtile.y,
 899                                                    rtile.w,
 900                                                    rtile.h,
 901                                                    rtile.offset,
 902                                                    rtile.stride,
 903                                                    (int*)queue_index.device_pointer,
 904                                                    dim.global_size[0] * dim.global_size[1],
 905                                                    (char*)use_queues_flags.device_pointer,
 906                                                    (uint*)work_pool_wgs.device_pointer,
 907                                                    rtile.num_samples,
 908                                                    (float*)rtile.buffer);
 909                 }
 910         }
 911
 912         return true;
 913 }
 914
 915 SplitKernelFunction* CPUSplitKernel::get_split_kernel_function(string kernel_name, const DeviceRequestedFeatures&)
 916 {
 917         CPUSplitKernelFunction *kernel = new CPUSplitKernelFunction(device);
 918
 919         kernel->func = device->split_kernels[kernel_name]();
 920         if(!kernel->func) {
 921                 delete kernel;
 922                 return NULL;
 923         }
 924
 925         return kernel;
 926 }
 927
 928 int2 CPUSplitKernel::split_kernel_local_size()
 929 {
 930         return make_int2(1, 1);
 931 }
 932
 933 int2 CPUSplitKernel::split_kernel_global_size(device_memory& /*kg*/, device_memory& /*data*/, DeviceTask * /*task*/) {
 934         return make_int2(1, 1);
 935 }
 936
 937 uint64_t CPUSplitKernel::state_buffer_size(device_memory& kernel_globals, device_memory& /*data*/, size_t num_threads) {
 938         KernelGlobals *kg = (KernelGlobals*)kernel_globals.device_pointer;
 939
 940         return split_data_buffer_size(kg, num_threads);
 941 }
 942
 943 Device *device_cpu_create(DeviceInfo& info, Stats &stats, bool background)
 944 {
 945         return new CPUDevice(info, stats, background);
 946 }
 947
 948 void device_cpu_info(vector<DeviceInfo>& devices)
 949 {
 950         DeviceInfo info;
 951
 952         info.type = DEVICE_CPU;
 953         info.description = system_cpu_brand_string();
 954         info.id = "CPU";
 955         info.num = 0;
 956         info.advanced_shading = true;
 957         info.pack_images = false;
 958
 959         devices.insert(devices.begin(), info);
 960 }
 961
 962 string device_cpu_capabilities(void)
 963 {
 964         string capabilities = "";
 965         capabilities += system_cpu_support_sse2() ? "SSE2 " : "";
 966         capabilities += system_cpu_support_sse3() ? "SSE3 " : "";
 967         capabilities += system_cpu_support_sse41() ? "SSE41 " : "";
 968         capabilities += system_cpu_support_avx() ? "AVX " : "";
 969         capabilities += system_cpu_support_avx2() ? "AVX2" : "";
 970         if(capabilities[capabilities.size() - 1] == ' ')
 971                 capabilities.resize(capabilities.size() - 1);
 972         return capabilities;
 973 }
 974
 975 CCL_NAMESPACE_END