Fix warnings and remove casts by adding copy_vx_vx_uchar() functions.
[blender.git] / source / blender / render / intern / source / bake.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * Contributors: 2004/2005/2006 Blender Foundation, full recode
19  * Contributors: Vertex color baking, Copyright 2011 AutoCRC
20  *
21  * ***** END GPL LICENSE BLOCK *****
22  */
23
24 /** \file blender/render/intern/source/bake.c
25  *  \ingroup render
26  */
27
28
29 /* system includes */
30 #include <stdio.h>
31 #include <string.h>
32
33 /* External modules: */
34 #include "MEM_guardedalloc.h"
35
36 #include "BLI_math.h"
37 #include "BLI_rand.h"
38 #include "BLI_threads.h"
39 #include "BLI_utildefines.h"
40
41 #include "DNA_image_types.h"
42 #include "DNA_material_types.h"
43 #include "DNA_mesh_types.h"
44 #include "DNA_meshdata_types.h"
45
46 #include "BKE_customdata.h"
47 #include "BKE_global.h"
48 #include "BKE_image.h"
49 #include "BKE_main.h"
50 #include "BKE_node.h"
51 #include "BKE_scene.h"
52 #include "BKE_library.h"
53
54 #include "IMB_imbuf_types.h"
55 #include "IMB_imbuf.h"
56 #include "IMB_colormanagement.h"
57
58 /* local include */
59 #include "rayintersection.h"
60 #include "rayobject.h"
61 #include "render_types.h"
62 #include "renderdatabase.h"
63 #include "shading.h"
64 #include "zbuf.h"
65
66 #include "PIL_time.h"
67
68 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
69 /* defined in pipeline.c, is hardcopy of active dynamic allocated Render */
70 /* only to be used here in this file, it's for speed */
71 extern struct Render R;
72 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
73
74
75 /* ************************* bake ************************ */
76
77
78 typedef struct BakeShade {
79         int thread;
80
81         ShadeSample ssamp;
82         ObjectInstanceRen *obi;
83         VlakRen *vlr;
84         
85         ZSpan *zspan;
86         Image *ima;
87         ImBuf *ibuf;
88         
89         int rectx, recty, quad, type, vdone;
90         bool ready;
91
92         float dir[3];
93         Object *actob;
94
95         /* Output: vertex color or image data. If vcol is not NULL, rect and
96          * rect_float should be NULL. */
97         MPoly *mpoly;
98         MLoop *mloop;
99         MLoopCol *vcol;
100         
101         unsigned int *rect;
102         float *rect_float;
103
104         /* displacement buffer used for normalization with unknown maximal distance */
105         bool use_displacement_buffer;
106         float *displacement_buffer;
107         float displacement_min, displacement_max;
108         
109         bool use_mask;
110         char *rect_mask; /* bake pixel mask */
111
112         float dxco[3], dyco[3];
113
114         short *do_update;
115
116         struct ColorSpace *rect_colorspace;
117 } BakeShade;
118
119 static void bake_set_shade_input(ObjectInstanceRen *obi, VlakRen *vlr, ShadeInput *shi, int quad, int UNUSED(isect), int x, int y, float u, float v)
120 {
121         if (quad)
122                 shade_input_set_triangle_i(shi, obi, vlr, 0, 2, 3);
123         else
124                 shade_input_set_triangle_i(shi, obi, vlr, 0, 1, 2);
125                 
126         /* cache for shadow */
127         shi->samplenr = R.shadowsamplenr[shi->thread]++;
128
129         shi->mask = 0xFFFF; /* all samples */
130
131         shi->u = -u;
132         shi->v = -v;
133         shi->xs = x;
134         shi->ys = y;
135
136         shade_input_set_uv(shi);
137         shade_input_set_normals(shi);
138
139         /* no normal flip */
140         if (shi->flippednor)
141                 shade_input_flip_normals(shi);
142
143         /* set up view vector to look right at the surface (note that the normal
144          * is negated in the renderer so it does not need to be done here) */
145         shi->view[0] = shi->vn[0];
146         shi->view[1] = shi->vn[1];
147         shi->view[2] = shi->vn[2];
148 }
149
150 static void bake_shade(void *handle, Object *ob, ShadeInput *shi, int UNUSED(quad), int x, int y, float UNUSED(u), float UNUSED(v), float *tvn, float *ttang)
151 {
152         BakeShade *bs = handle;
153         ShadeSample *ssamp = &bs->ssamp;
154         ShadeResult shr;
155         VlakRen *vlr = shi->vlr;
156
157         shade_input_init_material(shi);
158
159         if (bs->type == RE_BAKE_AO) {
160                 ambient_occlusion(shi);
161
162                 if (R.r.bake_flag & R_BAKE_NORMALIZE) {
163                         copy_v3_v3(shr.combined, shi->ao);
164                 }
165                 else {
166                         zero_v3(shr.combined);
167                         environment_lighting_apply(shi, &shr);
168                 }
169         }
170         else {
171                 if (bs->type == RE_BAKE_SHADOW) /* Why do shadows set the color anyhow?, ignore material color for baking */
172                         shi->r = shi->g = shi->b = 1.0f;
173
174                 shade_input_set_shade_texco(shi);
175                 
176                 /* only do AO for a full bake (and obviously AO bakes)
177                  * AO for light bakes is a leftover and might not be needed */
178                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_AO, RE_BAKE_LIGHT))
179                         shade_samples_do_AO(ssamp);
180                 
181                 if (shi->mat->nodetree && shi->mat->use_nodes) {
182                         ntreeShaderExecTree(shi->mat->nodetree, shi, &shr);
183                         shi->mat = vlr->mat;  /* shi->mat is being set in nodetree */
184                 }
185                 else
186                         shade_material_loop(shi, &shr);
187
188                 if (bs->type == RE_BAKE_NORMALS) {
189                         float nor[3];
190
191                         copy_v3_v3(nor, shi->vn);
192
193                         if (R.r.bake_normal_space == R_BAKE_SPACE_CAMERA) {
194                                 /* pass */
195                         }
196                         else if (R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
197                                 float mat[3][3], imat[3][3];
198
199                                 /* bitangent */
200                                 if (tvn && ttang) {
201                                         copy_v3_v3(mat[0], ttang);
202                                         cross_v3_v3v3(mat[1], tvn, ttang);
203                                         mul_v3_fl(mat[1], ttang[3]);
204                                         copy_v3_v3(mat[2], tvn);
205                                 }
206                                 else {
207                                         copy_v3_v3(mat[0], shi->nmaptang);
208                                         cross_v3_v3v3(mat[1], shi->nmapnorm, shi->nmaptang);
209                                         mul_v3_fl(mat[1], shi->nmaptang[3]);
210                                         copy_v3_v3(mat[2], shi->nmapnorm);
211                                 }
212
213                                 invert_m3_m3(imat, mat);
214                                 mul_m3_v3(imat, nor);
215                         }
216                         else if (R.r.bake_normal_space == R_BAKE_SPACE_OBJECT)
217                                 mul_mat3_m4_v3(ob->imat_ren, nor);  /* ob->imat_ren includes viewinv! */
218                         else if (R.r.bake_normal_space == R_BAKE_SPACE_WORLD)
219                                 mul_mat3_m4_v3(R.viewinv, nor);
220
221                         normalize_v3(nor); /* in case object has scaling */
222
223                         /* The invert of the red channel is to make
224                          * the normal map compliant with the outside world.
225                          * It needs to be done because in Blender
226                          * the normal used in the renderer points inward. It is generated
227                          * this way in calc_vertexnormals(). Should this ever change
228                          * this negate must be removed.
229                          *
230                          * there is also a small 1e-5f bias for precision issues. otherwise
231                          * we randomly get 127 or 128 for neutral colors. we choose 128
232                          * because it is the convention flat color. * */
233                         shr.combined[0] = (-nor[0]) / 2.0f + 0.5f + 1e-5f;
234                         shr.combined[1] = nor[1]    / 2.0f + 0.5f + 1e-5f;
235                         shr.combined[2] = nor[2]    / 2.0f + 0.5f + 1e-5f;
236                 }
237                 else if (bs->type == RE_BAKE_TEXTURE) {
238                         copy_v3_v3(shr.combined, &shi->r);
239                         shr.alpha = shi->alpha;
240                 }
241                 else if (bs->type == RE_BAKE_SHADOW) {
242                         copy_v3_v3(shr.combined, shr.shad);
243                         shr.alpha = shi->alpha;
244                 }
245                 else if (bs->type == RE_BAKE_SPEC_COLOR) {
246                         copy_v3_v3(shr.combined, &shi->specr);
247                         shr.alpha = 1.0f;
248                 }
249                 else if (bs->type == RE_BAKE_SPEC_INTENSITY) {
250                         copy_v3_fl(shr.combined, shi->spec);
251                         shr.alpha = 1.0f;
252                 }
253                 else if (bs->type == RE_BAKE_MIRROR_COLOR) {
254                         copy_v3_v3(shr.combined, &shi->mirr);
255                         shr.alpha = 1.0f;
256                 }
257                 else if (bs->type == RE_BAKE_MIRROR_INTENSITY) {
258                         copy_v3_fl(shr.combined, shi->ray_mirror);
259                         shr.alpha = 1.0f;
260                 }
261                 else if (bs->type == RE_BAKE_ALPHA) {
262                         copy_v3_fl(shr.combined, shi->alpha);
263                         shr.alpha = 1.0f;
264                 }
265                 else if (bs->type == RE_BAKE_EMIT) {
266                         copy_v3_fl(shr.combined, shi->emit);
267                         shr.alpha = 1.0f;
268                 }
269                 else if (bs->type == RE_BAKE_VERTEX_COLORS) {
270                         copy_v3_v3(shr.combined, shi->vcol);
271                         shr.alpha = shi->vcol[3];
272                 }
273         }
274         
275         if (bs->rect_float && !bs->vcol) {
276                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
277                 copy_v3_v3(col, shr.combined);
278                 if (bs->type == RE_BAKE_ALL || bs->type == RE_BAKE_TEXTURE || bs->type == RE_BAKE_VERTEX_COLORS) {
279                         col[3] = shr.alpha;
280                 }
281                 else {
282                         col[3] = 1.0;
283                 }
284         }
285         else {
286                 /* Target is char (LDR). */
287                 unsigned char col[4];
288
289                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
290                         float rgb[3];
291
292                         copy_v3_v3(rgb, shr.combined);
293                         if (R.scene_color_manage) {
294                                 /* Vertex colors have no way to specify color space, so they
295                                  * default to sRGB. */
296                                 if (!bs->vcol)
297                                         IMB_colormanagement_scene_linear_to_colorspace_v3(rgb, bs->rect_colorspace);
298                                 else
299                                         linearrgb_to_srgb_v3_v3(rgb, rgb);
300                         }
301                         rgb_float_to_uchar(col, rgb);
302                 }
303                 else {
304                         rgb_float_to_uchar(col, shr.combined);
305                 }
306                 
307                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE, RE_BAKE_VERTEX_COLORS)) {
308                         col[3] = FTOCHAR(shr.alpha);
309                 }
310                 else {
311                         col[3] = 255;
312                 }
313
314                 if (bs->vcol) {
315                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
316                          * but is used only for vertex painting). */
317                         bs->vcol->r = col[0];
318                         bs->vcol->g = col[1];
319                         bs->vcol->b = col[2];
320                 }
321                 else {
322                         unsigned char *imcol = (unsigned char *)(bs->rect + bs->rectx * y + x);
323                         copy_v4_v4_uchar(imcol, col);
324                 }
325
326         }
327         
328         if (bs->rect_mask) {
329                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
330         }
331
332         if (bs->do_update) {
333                 *bs->do_update = true;
334         }
335 }
336
337 static void bake_displacement(void *handle, ShadeInput *UNUSED(shi), float dist, int x, int y)
338 {
339         BakeShade *bs = handle;
340         float disp;
341
342         if (R.r.bake_flag & R_BAKE_NORMALIZE) {
343                 if (R.r.bake_maxdist)
344                         disp = (dist + R.r.bake_maxdist) / (R.r.bake_maxdist * 2);  /* alter the range from [-bake_maxdist, bake_maxdist] to [0, 1]*/
345                 else
346                         disp = dist;
347         }
348         else {
349                 disp = 0.5f + dist; /* alter the range from [-0.5,0.5] to [0,1]*/
350         }
351
352         if (bs->displacement_buffer) {
353                 float *displacement = bs->displacement_buffer + (bs->rectx * y + x);
354                 *displacement = disp;
355                 bs->displacement_min = min_ff(bs->displacement_min, disp);
356                 bs->displacement_max = max_ff(bs->displacement_max, disp);
357         }
358
359         if (bs->rect_float && !bs->vcol) {
360                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
361                 col[0] = col[1] = col[2] = disp;
362                 col[3] = 1.0f;
363         }
364         else {
365                 /* Target is char (LDR). */
366                 unsigned char col[4];
367                 col[0] = col[1] = col[2] = FTOCHAR(disp);
368                 col[3] = 255;
369
370                 if (bs->vcol) {
371                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
372                          * but is used only for vertex painting). */
373                         bs->vcol->r = col[0];
374                         bs->vcol->g = col[1];
375                         bs->vcol->b = col[2];
376                 }
377                 else {
378                         unsigned char *imcol = (unsigned char *)(bs->rect + bs->rectx * y + x);
379                         copy_v4_v4_uchar(imcol, col);
380                 }
381         }
382         if (bs->rect_mask) {
383                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
384         }
385 }
386
387 static int bake_intersect_tree(RayObject *raytree, Isect *isect, float *start, float *dir, float sign, float *hitco, float *dist)
388 {
389         float maxdist;
390         int hit;
391
392         /* might be useful to make a user setting for maxsize*/
393         if (R.r.bake_maxdist > 0.0f)
394                 maxdist = R.r.bake_maxdist;
395         else
396                 maxdist = RE_RAYTRACE_MAXDIST + R.r.bake_biasdist;
397
398         /* 'dir' is always normalized */
399         madd_v3_v3v3fl(isect->start, start, dir, -R.r.bake_biasdist);
400
401         mul_v3_v3fl(isect->dir, dir, sign);
402
403         isect->dist = maxdist;
404
405         hit = RE_rayobject_raycast(raytree, isect);
406         if (hit) {
407                 madd_v3_v3v3fl(hitco, isect->start, isect->dir, isect->dist);
408
409                 *dist = isect->dist;
410         }
411
412         return hit;
413 }
414
415 static void bake_set_vlr_dxyco(BakeShade *bs, float *uv1, float *uv2, float *uv3)
416 {
417         VlakRen *vlr = bs->vlr;
418         float A, d1, d2, d3, *v1, *v2, *v3;
419
420         if (bs->quad) {
421                 v1 = vlr->v1->co;
422                 v2 = vlr->v3->co;
423                 v3 = vlr->v4->co;
424         }
425         else {
426                 v1 = vlr->v1->co;
427                 v2 = vlr->v2->co;
428                 v3 = vlr->v3->co;
429         }
430
431         /* formula derived from barycentric coordinates:
432          * (uvArea1*v1 + uvArea2*v2 + uvArea3*v3)/uvArea
433          * then taking u and v partial derivatives to get dxco and dyco */
434         A = (uv2[0] - uv1[0]) * (uv3[1] - uv1[1]) - (uv3[0] - uv1[0]) * (uv2[1] - uv1[1]);
435
436         if (fabsf(A) > FLT_EPSILON) {
437                 A = 0.5f / A;
438
439                 d1 = uv2[1] - uv3[1];
440                 d2 = uv3[1] - uv1[1];
441                 d3 = uv1[1] - uv2[1];
442                 bs->dxco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
443                 bs->dxco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
444                 bs->dxco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
445
446                 d1 = uv3[0] - uv2[0];
447                 d2 = uv1[0] - uv3[0];
448                 d3 = uv2[0] - uv1[0];
449                 bs->dyco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
450                 bs->dyco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
451                 bs->dyco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
452         }
453         else {
454                 bs->dxco[0] = bs->dxco[1] = bs->dxco[2] = 0.0f;
455                 bs->dyco[0] = bs->dyco[1] = bs->dyco[2] = 0.0f;
456         }
457
458         if (bs->obi->flag & R_TRANSFORMED) {
459                 mul_m3_v3(bs->obi->nmat, bs->dxco);
460                 mul_m3_v3(bs->obi->nmat, bs->dyco);
461         }
462 }
463
464 static void do_bake_shade(void *handle, int x, int y, float u, float v)
465 {
466         BakeShade *bs = handle;
467         VlakRen *vlr = bs->vlr;
468         ObjectInstanceRen *obi = bs->obi;
469         Object *ob = obi->obr->ob;
470         float l, *v1, *v2, *v3, tvn[3], ttang[4];
471         int quad;
472         ShadeSample *ssamp = &bs->ssamp;
473         ShadeInput *shi = ssamp->shi;
474
475         /* fast threadsafe break test */
476         if (R.test_break(R.tbh))
477                 return;
478
479         /* setup render coordinates */
480         if (bs->quad) {
481                 v1 = vlr->v1->co;
482                 v2 = vlr->v3->co;
483                 v3 = vlr->v4->co;
484         }
485         else {
486                 v1 = vlr->v1->co;
487                 v2 = vlr->v2->co;
488                 v3 = vlr->v3->co;
489         }
490
491         l = 1.0f - u - v;
492
493         /* shrink barycentric coordinates inwards slightly to avoid some issues
494          * where baking selected to active might just miss the other face at the
495          * near the edge of a face */
496         if (bs->actob) {
497                 const float eps = 1.0f - 1e-4f;
498                 float invsum;
499
500                 u = (u - 0.5f) * eps + 0.5f;
501                 v = (v - 0.5f) * eps + 0.5f;
502                 l = (l - 0.5f) * eps + 0.5f;
503
504                 invsum = 1.0f / (u + v + l);
505
506                 u *= invsum;
507                 v *= invsum;
508                 l *= invsum;
509         }
510
511         /* renderco */
512         shi->co[0] = l * v3[0] + u * v1[0] + v * v2[0];
513         shi->co[1] = l * v3[1] + u * v1[1] + v * v2[1];
514         shi->co[2] = l * v3[2] + u * v1[2] + v * v2[2];
515
516         /* avoid self shadow with vertex bake from adjacent faces [#33729] */
517         if ((bs->vcol != NULL) && (bs->actob == NULL)) {
518                 madd_v3_v3fl(shi->co, vlr->n, 0.0001f);
519         }
520
521         if (obi->flag & R_TRANSFORMED)
522                 mul_m4_v3(obi->mat, shi->co);
523
524         copy_v3_v3(shi->dxco, bs->dxco);
525         copy_v3_v3(shi->dyco, bs->dyco);
526
527         quad = bs->quad;
528         bake_set_shade_input(obi, vlr, shi, quad, 0, x, y, u, v);
529
530         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
531                 shade_input_set_shade_texco(shi);
532                 copy_v3_v3(tvn, shi->nmapnorm);
533                 copy_v4_v4(ttang, shi->nmaptang);
534         }
535
536         /* if we are doing selected to active baking, find point on other face */
537         if (bs->actob) {
538                 Isect isec, minisec;
539                 float co[3], minco[3], dist, mindist = 0.0f;
540                 int hit, sign, dir = 1;
541
542                 /* intersect with ray going forward and backward*/
543                 hit = 0;
544                 memset(&minisec, 0, sizeof(minisec));
545                 minco[0] = minco[1] = minco[2] = 0.0f;
546
547                 copy_v3_v3(bs->dir, shi->vn);
548
549                 for (sign = -1; sign <= 1; sign += 2) {
550                         memset(&isec, 0, sizeof(isec));
551                         isec.mode = RE_RAY_MIRROR;
552
553                         isec.orig.ob   = obi;
554                         isec.orig.face = vlr;
555                         isec.userdata = bs->actob;
556                         isec.check = RE_CHECK_VLR_BAKE;
557                         isec.skip = RE_SKIP_VLR_NEIGHBOUR;
558
559                         if (bake_intersect_tree(R.raytree, &isec, shi->co, shi->vn, sign, co, &dist)) {
560                                 if (!hit || len_squared_v3v3(shi->co, co) < len_squared_v3v3(shi->co, minco)) {
561                                         minisec = isec;
562                                         mindist = dist;
563                                         copy_v3_v3(minco, co);
564                                         hit = 1;
565                                         dir = sign;
566                                 }
567                         }
568                 }
569
570                 if (ELEM(bs->type, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE)) {
571                         if (hit)
572                                 bake_displacement(handle, shi, (dir == -1) ? mindist : -mindist, x, y);
573                         else
574                                 bake_displacement(handle, shi, 0.0f, x, y);
575                         return;
576                 }
577
578                 /* if hit, we shade from the new point, otherwise from point one starting face */
579                 if (hit) {
580                         obi = (ObjectInstanceRen *)minisec.hit.ob;
581                         vlr = (VlakRen *)minisec.hit.face;
582                         quad = (minisec.isect == 2);
583                         copy_v3_v3(shi->co, minco);
584
585                         u = -minisec.u;
586                         v = -minisec.v;
587                         bake_set_shade_input(obi, vlr, shi, quad, 1, x, y, u, v);
588                 }
589         }
590
591         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT)
592                 bake_shade(handle, ob, shi, quad, x, y, u, v, tvn, ttang);
593         else
594                 bake_shade(handle, ob, shi, quad, x, y, u, v, NULL, NULL);
595 }
596
597 static int get_next_bake_face(BakeShade *bs)
598 {
599         ObjectRen *obr;
600         VlakRen *vlr;
601         MTFace *tface;
602         static int v = 0, vdone = false;
603         static ObjectInstanceRen *obi = NULL;
604
605         if (bs == NULL) {
606                 vlr = NULL;
607                 v = vdone = false;
608                 obi = R.instancetable.first;
609                 return 0;
610         }
611         
612         BLI_lock_thread(LOCK_CUSTOM1);
613
614         for (; obi; obi = obi->next, v = 0) {
615                 obr = obi->obr;
616
617                 /* only allow non instances here */
618                 if (obr->flag & R_INSTANCEABLE)
619                         continue;
620
621                 for (; v < obr->totvlak; v++) {
622                         vlr = RE_findOrAddVlak(obr, v);
623
624                         if ((bs->actob && bs->actob == obr->ob) || (!bs->actob && (obr->ob->flag & SELECT))) {
625                                 if (R.r.bake_flag & R_BAKE_VCOL) {
626                                         /* Gather face data for vertex color bake */
627                                         Mesh *me;
628                                         int *origindex, vcollayer;
629                                         CustomDataLayer *cdl;
630
631                                         if (obr->ob->type != OB_MESH)
632                                                 continue;
633                                         me = obr->ob->data;
634
635                                         origindex = RE_vlakren_get_origindex(obr, vlr, 0);
636                                         if (origindex == NULL)
637                                                 continue;
638                                         if (*origindex >= me->totpoly) {
639                                                 /* Small hack for Array modifier, which gives false
640                                                  * original indices - z0r */
641                                                 continue;
642                                         }
643 #if 0
644                                         /* Only shade selected faces. */
645                                         if ((me->mface[*origindex].flag & ME_FACE_SEL) == 0)
646                                                 continue;
647 #endif
648
649                                         vcollayer = CustomData_get_render_layer_index(&me->ldata, CD_MLOOPCOL);
650                                         if (vcollayer == -1)
651                                                 continue;
652
653                                         cdl = &me->ldata.layers[vcollayer];
654                                         bs->mpoly = me->mpoly + *origindex;
655                                         bs->vcol = ((MLoopCol *)cdl->data) + bs->mpoly->loopstart;
656                                         bs->mloop = me->mloop + bs->mpoly->loopstart;
657
658                                         /* Tag mesh for reevaluation. */
659                                         me->id.flag |= LIB_DOIT;
660                                 }
661                                 else {
662                                         Image *ima = NULL;
663                                         ImBuf *ibuf = NULL;
664                                         const float vec_alpha[4] = {0.0f, 0.0f, 0.0f, 0.0f};
665                                         const float vec_solid[4] = {0.0f, 0.0f, 0.0f, 1.0f};
666                                         const float nor_alpha[4] = {0.5f, 0.5f, 1.0f, 0.0f};
667                                         const float nor_solid[4] = {0.5f, 0.5f, 1.0f, 1.0f};
668                                         const float disp_alpha[4] = {0.5f, 0.5f, 0.5f, 0.0f};
669                                         const float disp_solid[4] = {0.5f, 0.5f, 0.5f, 1.0f};
670
671                                         tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
672
673                                         if (!tface || !tface->tpage)
674                                                 continue;
675
676                                         ima = tface->tpage;
677                                         ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
678
679                                         if (ibuf == NULL)
680                                                 continue;
681
682                                         if (ibuf->rect == NULL && ibuf->rect_float == NULL) {
683                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
684                                                 continue;
685                                         }
686
687                                         if (ibuf->rect_float && !(ibuf->channels == 0 || ibuf->channels == 4)) {
688                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
689                                                 continue;
690                                         }
691                                         
692                                         if (ima->flag & IMA_USED_FOR_RENDER) {
693                                                 ima->id.flag &= ~LIB_DOIT;
694                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
695                                                 continue;
696                                         }
697                                         
698                                         /* find the image for the first time? */
699                                         if (ima->id.flag & LIB_DOIT) {
700                                                 ima->id.flag &= ~LIB_DOIT;
701                                                 
702                                                 /* we either fill in float or char, this ensures things go fine */
703                                                 if (ibuf->rect_float)
704                                                         imb_freerectImBuf(ibuf);
705                                                 /* clear image */
706                                                 if (R.r.bake_flag & R_BAKE_CLEAR) {
707                                                         if (R.r.bake_mode == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT)
708                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? nor_alpha : nor_solid);
709                                                         else if (ELEM(R.r.bake_mode, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE))
710                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? disp_alpha : disp_solid);
711                                                         else
712                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? vec_alpha : vec_solid);
713                                                 }
714                                                 /* might be read by UI to set active image for display */
715                                                 R.bakebuf = ima;
716                                         }
717
718                                         /* Tag image for redraw. */
719                                         ibuf->userflags |= IB_DISPLAY_BUFFER_INVALID;
720                                         BKE_image_release_ibuf(ima, ibuf, NULL);
721                                 }
722
723                                 bs->obi = obi;
724                                 bs->vlr = vlr;
725                                 bs->vdone++;  /* only for error message if nothing was rendered */
726                                 v++;
727                                 BLI_unlock_thread(LOCK_CUSTOM1);
728                                 return 1;
729                         }
730                 }
731         }
732         
733         BLI_unlock_thread(LOCK_CUSTOM1);
734         return 0;
735 }
736
737 static void bake_single_vertex(BakeShade *bs, VertRen *vert, float u, float v)
738 {
739         int *origindex, i;
740         MLoopCol *basevcol;
741         MLoop *mloop;
742
743         /* per vertex fixed seed */
744         BLI_thread_srandom(bs->thread, vert->index);
745
746         origindex = RE_vertren_get_origindex(bs->obi->obr, vert, 0);
747         if (!origindex || *origindex == ORIGINDEX_NONE)
748                 return;
749
750         /* Search for matching vertex index and apply shading. */
751         for (i = 0; i < bs->mpoly->totloop; i++) {
752                 mloop = bs->mloop + i;
753                 if (mloop->v != *origindex)
754                         continue;
755                 basevcol = bs->vcol;
756                 bs->vcol = basevcol + i;
757                 do_bake_shade(bs, 0, 0, u, v);
758                 bs->vcol = basevcol;
759                 break;
760         }
761 }
762
763 /* Bake all vertices of a face. Actually, this still works on a face-by-face
764  * basis, and each vertex on each face is shaded. Vertex colors are a property
765  * of loops, not vertices. */
766 static void shade_verts(BakeShade *bs)
767 {
768         VlakRen *vlr = bs->vlr;
769
770         /* Disable baking to image; write to vcol instead. vcol pointer is set in
771          * bake_single_vertex. */
772         bs->ima = NULL;
773         bs->rect = NULL;
774         bs->rect_float = NULL;
775         bs->displacement_buffer = NULL;
776         bs->displacement_min = FLT_MAX;
777         bs->displacement_max = -FLT_MAX;
778
779         bs->quad = 0;
780
781         /* No anti-aliasing for vertices. */
782         zero_v3(bs->dxco);
783         zero_v3(bs->dyco);
784
785         /* Shade each vertex of the face. u and v are barycentric coordinates; since
786          * we're only interested in vertices, these will be 0 or 1. */
787         if ((vlr->flag & R_FACE_SPLIT) == 0) {
788                 /* Processing triangle face, whole quad, or first half of split quad. */
789
790                 bake_single_vertex(bs, bs->vlr->v1, 1.0f, 0.0f);
791                 bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
792                 bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
793
794                 if (vlr->v4) {
795                         bs->quad = 1;
796                         bake_single_vertex(bs, bs->vlr->v4, 0.0f, 0.0f);
797                 }
798         }
799         else {
800                 /* Processing second half of split quad. Only one vertex to go. */
801                 if (vlr->flag & R_DIVIDE_24) {
802                         bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
803                 }
804                 else {
805                         bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
806                 }
807         }
808 }
809
810 /* already have tested for tface and ima and zspan */
811 static void shade_tface(BakeShade *bs)
812 {
813         VlakRen *vlr = bs->vlr;
814         ObjectInstanceRen *obi = bs->obi;
815         ObjectRen *obr = obi->obr;
816         MTFace *tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
817         Image *ima = tface->tpage;
818         float vec[4][2];
819         int a, i1, i2, i3;
820
821         /* per face fixed seed */
822         BLI_thread_srandom(bs->thread, vlr->index);
823         
824         /* check valid zspan */
825         if (ima != bs->ima) {
826                 BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
827
828                 bs->ima = ima;
829                 bs->ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
830                 /* note, these calls only free/fill contents of zspan struct, not zspan itself */
831                 zbuf_free_span(bs->zspan);
832                 zbuf_alloc_span(bs->zspan, bs->ibuf->x, bs->ibuf->y, R.clipcrop);
833         }
834
835         bs->rectx = bs->ibuf->x;
836         bs->recty = bs->ibuf->y;
837         bs->rect = bs->ibuf->rect;
838         bs->rect_colorspace = bs->ibuf->rect_colorspace;
839         bs->rect_float = bs->ibuf->rect_float;
840         bs->vcol = NULL;
841         bs->quad = 0;
842         bs->rect_mask = NULL;
843         bs->displacement_buffer = NULL;
844
845         if (bs->use_mask || bs->use_displacement_buffer) {
846                 BakeImBufuserData *userdata = bs->ibuf->userdata;
847                 if (userdata == NULL) {
848                         BLI_lock_thread(LOCK_CUSTOM1);
849                         userdata = bs->ibuf->userdata;
850                         if (userdata == NULL) /* since the thread was locked, its possible another thread alloced the value */
851                                 userdata = MEM_callocN(sizeof(BakeImBufuserData), "BakeImBufuserData");
852
853                         if (bs->use_mask) {
854                                 if (userdata->mask_buffer == NULL) {
855                                         userdata->mask_buffer = MEM_callocN(sizeof(char) * bs->rectx * bs->recty, "BakeMask");
856                                 }
857                         }
858
859                         if (bs->use_displacement_buffer) {
860                                 if (userdata->displacement_buffer == NULL) {
861                                         userdata->displacement_buffer = MEM_callocN(sizeof(float) * bs->rectx * bs->recty, "BakeDisp");
862                                 }
863                         }
864
865                         bs->ibuf->userdata = userdata;
866
867                         BLI_unlock_thread(LOCK_CUSTOM1);
868                 }
869
870                 bs->rect_mask = userdata->mask_buffer;
871                 bs->displacement_buffer = userdata->displacement_buffer;
872         }
873         
874         /* get pixel level vertex coordinates */
875         for (a = 0; a < 4; a++) {
876                 /* Note, workaround for pixel aligned UVs which are common and can screw up our intersection tests
877                  * where a pixel gets in between 2 faces or the middle of a quad,
878                  * camera aligned quads also have this problem but they are less common.
879                  * Add a small offset to the UVs, fixes bug #18685 - Campbell */
880                 vec[a][0] = tface->uv[a][0] * (float)bs->rectx - (0.5f + 0.001f);
881                 vec[a][1] = tface->uv[a][1] * (float)bs->recty - (0.5f + 0.002f);
882         }
883
884         /* UV indices have to be corrected for possible quad->tria splits */
885         i1 = 0; i2 = 1; i3 = 2;
886         vlr_set_uv_indices(vlr, &i1, &i2, &i3);
887         bake_set_vlr_dxyco(bs, vec[i1], vec[i2], vec[i3]);
888         zspan_scanconvert(bs->zspan, bs, vec[i1], vec[i2], vec[i3], do_bake_shade);
889         
890         if (vlr->v4) {
891                 bs->quad = 1;
892                 bake_set_vlr_dxyco(bs, vec[0], vec[2], vec[3]);
893                 zspan_scanconvert(bs->zspan, bs, vec[0], vec[2], vec[3], do_bake_shade);
894         }
895 }
896
897 static void *do_bake_thread(void *bs_v)
898 {
899         BakeShade *bs = bs_v;
900
901         while (get_next_bake_face(bs)) {
902                 if (R.r.bake_flag & R_BAKE_VCOL) {
903                         shade_verts(bs);
904                 }
905                 else {
906                         shade_tface(bs);
907                 }
908                 
909                 /* fast threadsafe break test */
910                 if (R.test_break(R.tbh))
911                         break;
912
913                 /* access is not threadsafe but since its just true/false probably ok
914                  * only used for interactive baking */
915                 if (bs->do_update) {
916                         *bs->do_update = true;
917                 }
918         }
919         bs->ready = true;
920
921         BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
922
923         return NULL;
924 }
925
926 void RE_bake_ibuf_filter(ImBuf *ibuf, char *mask, const int filter)
927 {
928         /* must check before filtering */
929         const short is_new_alpha = (ibuf->planes != R_IMF_PLANES_RGBA) && BKE_imbuf_alpha_test(ibuf);
930
931         /* Margin */
932         if (filter) {
933                 IMB_filter_extend(ibuf, mask, filter);
934         }
935
936         /* if the bake results in new alpha then change the image setting */
937         if (is_new_alpha) {
938                 ibuf->planes = R_IMF_PLANES_RGBA;
939         }
940         else {
941                 if (filter && ibuf->planes != R_IMF_PLANES_RGBA) {
942                         /* clear alpha added by filtering */
943                         IMB_rectfill_alpha(ibuf, 1.0f);
944                 }
945         }
946 }
947
948 void RE_bake_ibuf_normalize_displacement(ImBuf *ibuf, float *displacement, char *mask, float displacement_min, float displacement_max)
949 {
950         int i;
951         const float *current_displacement = displacement;
952         const char *current_mask = mask;
953         float max_distance;
954
955         max_distance = max_ff(fabsf(displacement_min), fabsf(displacement_max));
956
957         for (i = 0; i < ibuf->x * ibuf->y; i++) {
958                 if (*current_mask == FILTER_MASK_USED) {
959                         float normalized_displacement;
960
961                         if (max_distance > 1e-5f)
962                                 normalized_displacement = (*current_displacement + max_distance) / (max_distance * 2);
963                         else
964                                 normalized_displacement = 0.5f;
965
966                         if (ibuf->rect_float) {
967                                 /* currently baking happens to RGBA only */
968                                 float *fp = ibuf->rect_float + i * 4;
969                                 fp[0] = fp[1] = fp[2] = normalized_displacement;
970                                 fp[3] = 1.0f;
971                         }
972
973                         if (ibuf->rect) {
974                                 unsigned char *cp = (unsigned char *) (ibuf->rect + i);
975                                 cp[0] = cp[1] = cp[2] = FTOCHAR(normalized_displacement);
976                                 cp[3] = 255;
977                         }
978                 }
979
980                 current_displacement++;
981                 current_mask++;
982         }
983 }
984
985 /* using object selection tags, the faces with UV maps get baked */
986 /* render should have been setup */
987 /* returns 0 if nothing was handled */
988 int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_update, float *progress)
989 {
990         BakeShade *handles;
991         ListBase threads;
992         Image *ima;
993         int a, vdone = false, result = BAKE_RESULT_OK;
994         bool use_mask = false;
995         bool use_displacement_buffer = false;
996         bool do_manage = false;
997
998         if (ELEM(type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
999                 do_manage = BKE_scene_check_color_management_enabled(re->scene);
1000         }
1001
1002         re->scene_color_manage = BKE_scene_check_color_management_enabled(re->scene);
1003         
1004         /* initialize render global */
1005         R = *re;
1006         R.bakebuf = NULL;
1007
1008         /* initialize static vars */
1009         get_next_bake_face(NULL);
1010         
1011         /* do we need a mask? */
1012         if (re->r.bake_filter)
1013                 use_mask = true;
1014
1015         /* do we need buffer to store displacements  */
1016         if (ELEM(type, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE)) {
1017                 if (((R.r.bake_flag & R_BAKE_NORMALIZE) && R.r.bake_maxdist == 0.0f) ||
1018                     (type == RE_BAKE_DERIVATIVE))
1019                 {
1020                         use_displacement_buffer = true;
1021                         use_mask = true;
1022                 }
1023         }
1024
1025         /* baker uses this flag to detect if image was initialized */
1026         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
1027                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
1028                         ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
1029                         ima->id.flag |= LIB_DOIT;
1030                         ima->flag &= ~IMA_USED_FOR_RENDER;
1031                         if (ibuf) {
1032                                 ibuf->userdata = NULL; /* use for masking if needed */
1033                         }
1034                         BKE_image_release_ibuf(ima, ibuf, NULL);
1035                 }
1036         }
1037
1038         if (R.r.bake_flag & R_BAKE_VCOL) {
1039                 /* untag all meshes */
1040                 BKE_main_id_tag_listbase(&G.main->mesh, false);
1041         }
1042
1043         BLI_init_threads(&threads, do_bake_thread, re->r.threads);
1044
1045         handles = MEM_callocN(sizeof(BakeShade) * re->r.threads, "BakeShade");
1046
1047         /* get the threads running */
1048         for (a = 0; a < re->r.threads; a++) {
1049                 handles[a].thread = a;
1050
1051                 /* set defaults in handles */
1052                 handles[a].ssamp.shi[0].lay = re->lay;
1053
1054                 if (type == RE_BAKE_SHADOW) {
1055                         handles[a].ssamp.shi[0].passflag = SCE_PASS_SHADOW;
1056                 }
1057                 else {
1058                         handles[a].ssamp.shi[0].passflag = SCE_PASS_COMBINED;
1059                 }
1060                 handles[a].ssamp.shi[0].combinedflag = ~(SCE_PASS_SPEC);
1061                 handles[a].ssamp.shi[0].thread = a;
1062                 handles[a].ssamp.shi[0].do_manage = do_manage;
1063                 handles[a].ssamp.tot = 1;
1064
1065                 handles[a].type = type;
1066                 handles[a].actob = actob;
1067                 if (R.r.bake_flag & R_BAKE_VCOL)
1068                         handles[a].zspan = NULL;
1069                 else
1070                         handles[a].zspan = MEM_callocN(sizeof(ZSpan), "zspan for bake");
1071                 
1072                 handles[a].use_mask = use_mask;
1073                 handles[a].use_displacement_buffer = use_displacement_buffer;
1074
1075                 handles[a].do_update = do_update; /* use to tell the view to update */
1076                 
1077                 handles[a].displacement_min = FLT_MAX;
1078                 handles[a].displacement_max = -FLT_MAX;
1079
1080                 BLI_insert_thread(&threads, &handles[a]);
1081         }
1082         
1083         /* wait for everything to be done */
1084         a = 0;
1085         while (a != re->r.threads) {
1086                 PIL_sleep_ms(50);
1087
1088                 /* calculate progress */
1089                 for (vdone = false, a = 0; a < re->r.threads; a++)
1090                         vdone += handles[a].vdone;
1091                 if (progress)
1092                         *progress = (float)(vdone / (float)re->totvlak);
1093
1094                 for (a = 0; a < re->r.threads; a++) {
1095                         if (handles[a].ready == false) {
1096                                 break;
1097                         }
1098                 }
1099         }
1100
1101         /* filter and refresh images */
1102         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
1103                 float displacement_min = FLT_MAX, displacement_max = -FLT_MAX;
1104
1105                 if (use_displacement_buffer) {
1106                         for (a = 0; a < re->r.threads; a++) {
1107                                 displacement_min = min_ff(displacement_min, handles[a].displacement_min);
1108                                 displacement_max = max_ff(displacement_max, handles[a].displacement_max);
1109                         }
1110                 }
1111
1112                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
1113                         if ((ima->id.flag & LIB_DOIT) == 0) {
1114                                 ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
1115                                 BakeImBufuserData *userdata;
1116
1117                                 if (ima->flag & IMA_USED_FOR_RENDER)
1118                                         result = BAKE_RESULT_FEEDBACK_LOOP;
1119
1120                                 if (!ibuf)
1121                                         continue;
1122
1123                                 userdata = (BakeImBufuserData *)ibuf->userdata;
1124                                 if (userdata) {
1125                                         if (use_displacement_buffer) {
1126                                                 if (type == RE_BAKE_DERIVATIVE) {
1127                                                         float user_scale = (R.r.bake_flag & R_BAKE_USERSCALE) ? R.r.bake_user_scale : -1.0f;
1128                                                         RE_bake_make_derivative(ibuf, userdata->displacement_buffer, userdata->mask_buffer,
1129                                                                                 displacement_min, displacement_max, user_scale);
1130                                                 }
1131                                                 else {
1132                                                         RE_bake_ibuf_normalize_displacement(ibuf, userdata->displacement_buffer, userdata->mask_buffer,
1133                                                                                             displacement_min, displacement_max);
1134                                                 }
1135                                         }
1136
1137                                         RE_bake_ibuf_filter(ibuf, userdata->mask_buffer, re->r.bake_filter);
1138                                 }
1139
1140                                 ibuf->userflags |= IB_BITMAPDIRTY;
1141                                 BKE_image_release_ibuf(ima, ibuf, NULL);
1142                         }
1143                 }
1144
1145                 /* calculate return value */
1146                 for (a = 0; a < re->r.threads; a++) {
1147                         zbuf_free_span(handles[a].zspan);
1148                         MEM_freeN(handles[a].zspan);
1149                 }
1150         }
1151
1152         MEM_freeN(handles);
1153         
1154         BLI_end_threads(&threads);
1155
1156         if (vdone == 0) {
1157                 result = BAKE_RESULT_NO_OBJECTS;
1158         }
1159
1160         return result;
1161 }
1162
1163 struct Image *RE_bake_shade_get_image(void)
1164 {
1165         return R.bakebuf;
1166 }
1167
1168 /* **************** Derivative Maps Baker **************** */
1169
1170 static void add_single_heights_margin(const ImBuf *ibuf, const char *mask, float *heights_buffer)
1171 {
1172         int x, y;
1173
1174         for (y = 0; y < ibuf->y; y++) {
1175                 for (x = 0; x < ibuf->x; x++) {
1176                         int index = ibuf->x * y + x;
1177
1178                         /* If unassigned pixel, look for neighbors. */
1179                         if (mask[index] != FILTER_MASK_USED) {
1180                                 float height_acc = 0;
1181                                 int denom = 0;
1182                                 int i, j;
1183
1184                                 for (j = -1; j <= 1; j++)
1185                                         for (i = -1; i <= 1; i++) {
1186                                                 int w = (i == 0 ? 1 : 0) + (j == 0 ? 1 : 0) + 1;
1187
1188                                                 if (i != 0 || j != 0) {
1189                                                         int index2 = 0;
1190                                                         int x0 = x + i;
1191                                                         int y0 = y + j;
1192
1193                                                         CLAMP(x0, 0, ibuf->x - 1);
1194                                                         CLAMP(y0, 0, ibuf->y - 1);
1195
1196                                                         index2 = ibuf->x * y0 + x0;
1197
1198                                                         if (mask[index2] == FILTER_MASK_USED) {
1199                                                                 height_acc += w * heights_buffer[index2];
1200                                                                 denom += w;
1201                                                         }
1202                                                 }
1203                                         }
1204
1205                                 /* Insert final value. */
1206                                 if (denom > 0) {
1207                                         heights_buffer[index] = height_acc / denom;
1208                                 }
1209                         }
1210                 }
1211         }
1212 }
1213
1214 /* returns user-scale */
1215 float RE_bake_make_derivative(ImBuf *ibuf, float *heights_buffer, const char *mask,
1216                               const float height_min, const float height_max,
1217                               const float fmult)
1218 {
1219         const float delta_height = height_max - height_min;
1220         const float denom = delta_height > 0.0f ? (8 * delta_height) : 1.0f;
1221         bool auto_range_fit = fmult <= 0.0f;
1222         float max_num_deriv = -1.0f;
1223         int x, y, index;
1224
1225         /* Need a single margin to calculate good derivatives. */
1226         add_single_heights_margin(ibuf, mask, heights_buffer);
1227
1228         if (auto_range_fit) {
1229                 /* If automatic range fitting is enabled. */
1230                 for (y = 0; y < ibuf->y; y++) {
1231                         const int Yu = y == (ibuf->y - 1) ? (ibuf->y - 1) : (y + 1);
1232                         const int Yc = y;
1233                         const int Yd = y == 0 ? 0 : (y - 1);
1234
1235                         for (x = 0; x < ibuf->x; x++) {
1236                                 const int Xl = x == 0 ? 0 : (x - 1);
1237                                 const int Xc = x;
1238                                 const int Xr = x == (ibuf->x - 1) ? (ibuf->x - 1) : (x + 1);
1239
1240                                 const float Hcy = heights_buffer[Yc * ibuf->x + Xr] - heights_buffer[Yc * ibuf->x + Xl];
1241                                 const float Hu  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yu * ibuf->x + Xl];
1242                                 const float Hd  = heights_buffer[Yd * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xl];
1243
1244                                 const float Hl  = heights_buffer[Yu * ibuf->x + Xl] - heights_buffer[Yd * ibuf->x + Xl];
1245                                 const float Hcx = heights_buffer[Yu * ibuf->x + Xc] - heights_buffer[Yd * ibuf->x + Xc];
1246                                 const float Hr  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xr];
1247
1248                                 /* This corresponds to using the sobel kernel on the heights buffer
1249                                  * to obtain the derivative multiplied by 8.
1250                                  */
1251                                 const float deriv_x = Hu + 2 * Hcy + Hd;
1252                                 const float deriv_y = Hr + 2 * Hcx + Hl;
1253
1254                                 /* early out */
1255                                 index = ibuf->x * y + x;
1256                                 if (mask[index] != FILTER_MASK_USED) {
1257                                         continue;
1258                                 }
1259
1260                                 /* Widen bound. */
1261                                 if (fabsf(deriv_x) > max_num_deriv) {
1262                                         max_num_deriv = fabsf(deriv_x);
1263                                 }
1264
1265                                 if (fabsf(deriv_y) > max_num_deriv) {
1266                                         max_num_deriv = fabsf(deriv_y);
1267                                 }
1268                         }
1269                 }
1270         }
1271
1272         /* Output derivatives. */
1273         auto_range_fit &= (max_num_deriv > 0);
1274         for (y = 0; y < ibuf->y; y++) {
1275                 const int Yu = y == (ibuf->y - 1) ? (ibuf->y - 1) : (y + 1);
1276                 const int Yc = y;
1277                 const int Yd = y == 0 ? 0 : (y - 1);
1278
1279                 for (x = 0; x < ibuf->x; x++) {
1280                         const int Xl = x == 0 ? 0 : (x - 1);
1281                         const int Xc = x;
1282                         const int Xr = x == (ibuf->x - 1) ? (ibuf->x - 1) : (x + 1);
1283
1284                         const float Hcy = heights_buffer[Yc * ibuf->x + Xr] - heights_buffer[Yc * ibuf->x + Xl];
1285                         const float Hu  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yu * ibuf->x + Xl];
1286                         const float Hd  = heights_buffer[Yd * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xl];
1287
1288                         const float Hl  = heights_buffer[Yu * ibuf->x + Xl] - heights_buffer[Yd * ibuf->x + Xl];
1289                         const float Hcx = heights_buffer[Yu * ibuf->x + Xc] - heights_buffer[Yd * ibuf->x + Xc];
1290                         const float Hr  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xr];
1291
1292                         /* This corresponds to using the sobel kernel on the heights buffer
1293                          * to obtain the derivative multiplied by 8.
1294                          */
1295                         float deriv_x = Hu + 2 * Hcy + Hd;
1296                         float deriv_y = Hr + 2 * Hcx + Hl;
1297
1298                         /* Early out. */
1299                         index = ibuf->x * y + x;
1300                         if (mask[index] != FILTER_MASK_USED) {
1301                                 continue;
1302                         }
1303
1304                         if (auto_range_fit) {
1305                                 deriv_x /= max_num_deriv;
1306                                 deriv_y /= max_num_deriv;
1307                         }
1308                         else {
1309                                 deriv_x *= (fmult / denom);
1310                                 deriv_y *= (fmult / denom);
1311                         }
1312
1313                         deriv_x = deriv_x * 0.5f + 0.5f;
1314                         deriv_y = deriv_y * 0.5f + 0.5f;
1315
1316                         /* Clamp. */
1317                         CLAMP(deriv_x, 0.0f, 1.0f);
1318                         CLAMP(deriv_y, 0.0f, 1.0f);
1319
1320                         /* Write out derivatives. */
1321                         if (ibuf->rect_float) {
1322                                 float *rrgbf = ibuf->rect_float + index * 4;
1323
1324                                 rrgbf[0] = deriv_x;
1325                                 rrgbf[1] = deriv_y;
1326                                 rrgbf[2] = 0.0f;
1327                                 rrgbf[3] = 1.0f;
1328                         }
1329                         else {
1330                                 char *rrgb = (char *)ibuf->rect + index * 4;
1331
1332                                 rrgb[0] = FTOCHAR(deriv_x);
1333                                 rrgb[1] = FTOCHAR(deriv_y);
1334                                 rrgb[2] = 0;
1335                                 rrgb[3] = 255;
1336                         }
1337                 }
1338         }
1339
1340         /* Eeturn user-scale (for rendering). */
1341         return auto_range_fit ? (max_num_deriv / denom) : (fmult > 0.0f ? (1.0f / fmult) : 0.0f);
1342 }