Support for baking vertex colors to textures.
[blender-staging.git] / source / blender / render / intern / source / bake.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * Contributors: 2004/2005/2006 Blender Foundation, full recode
19  * Contributors: Vertex color baking, Copyright 2011 AutoCRC
20  *
21  * ***** END GPL LICENSE BLOCK *****
22  */
23
24 /** \file blender/render/intern/source/bake.c
25  *  \ingroup render
26  */
27
28
29 /* system includes */
30 #include <stdio.h>
31 #include <string.h>
32
33 /* External modules: */
34 #include "MEM_guardedalloc.h"
35
36 #include "BLI_math.h"
37 #include "BLI_blenlib.h"
38 #include "BLI_threads.h"
39 #include "BLI_utildefines.h"
40
41 #include "DNA_image_types.h"
42 #include "DNA_material_types.h"
43 #include "DNA_mesh_types.h"
44 #include "DNA_meshdata_types.h"
45
46 #include "BKE_customdata.h"
47 #include "BKE_global.h"
48 #include "BKE_image.h"
49 #include "BKE_main.h"
50 #include "BKE_node.h"
51 #include "BKE_scene.h"
52 #include "BKE_library.h"
53
54 #include "IMB_imbuf_types.h"
55 #include "IMB_imbuf.h"
56 #include "IMB_colormanagement.h"
57
58 /* local include */
59 #include "rayintersection.h"
60 #include "rayobject.h"
61 #include "render_types.h"
62 #include "renderdatabase.h"
63 #include "shading.h"
64 #include "zbuf.h"
65
66 #include "PIL_time.h"
67
68 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
69 /* defined in pipeline.c, is hardcopy of active dynamic allocated Render */
70 /* only to be used here in this file, it's for speed */
71 extern struct Render R;
72 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
73
74
75 /* ************************* bake ************************ */
76
77
78 typedef struct BakeShade {
79         ShadeSample ssamp;
80         ObjectInstanceRen *obi;
81         VlakRen *vlr;
82         
83         ZSpan *zspan;
84         Image *ima;
85         ImBuf *ibuf;
86         
87         int rectx, recty, quad, type, vdone;
88         bool ready;
89
90         float dir[3];
91         Object *actob;
92
93         /* Output: vertex color or image data. If vcol is not NULL, rect and
94          * rect_float should be NULL. */
95         MPoly *mpoly;
96         MLoop *mloop;
97         MLoopCol *vcol;
98         
99         unsigned int *rect;
100         float *rect_float;
101
102         /* displacement buffer used for normalization with unknown maximal distance */
103         bool use_displacement_buffer;
104         float *displacement_buffer;
105         float displacement_min, displacement_max;
106         
107         bool use_mask;
108         char *rect_mask; /* bake pixel mask */
109
110         float dxco[3], dyco[3];
111
112         short *do_update;
113
114         struct ColorSpace *rect_colorspace;
115 } BakeShade;
116
117 static void bake_set_shade_input(ObjectInstanceRen *obi, VlakRen *vlr, ShadeInput *shi, int quad, int UNUSED(isect), int x, int y, float u, float v)
118 {
119         if (quad)
120                 shade_input_set_triangle_i(shi, obi, vlr, 0, 2, 3);
121         else
122                 shade_input_set_triangle_i(shi, obi, vlr, 0, 1, 2);
123                 
124         /* cache for shadow */
125         shi->samplenr = R.shadowsamplenr[shi->thread]++;
126
127         shi->mask = 0xFFFF; /* all samples */
128
129         shi->u = -u;
130         shi->v = -v;
131         shi->xs = x;
132         shi->ys = y;
133
134         shade_input_set_uv(shi);
135         shade_input_set_normals(shi);
136
137         /* no normal flip */
138         if (shi->flippednor)
139                 shade_input_flip_normals(shi);
140
141         /* set up view vector to look right at the surface (note that the normal
142          * is negated in the renderer so it does not need to be done here) */
143         shi->view[0] = shi->vn[0];
144         shi->view[1] = shi->vn[1];
145         shi->view[2] = shi->vn[2];
146 }
147
148 static void bake_shade(void *handle, Object *ob, ShadeInput *shi, int UNUSED(quad), int x, int y, float UNUSED(u), float UNUSED(v), float *tvn, float *ttang)
149 {
150         BakeShade *bs = handle;
151         ShadeSample *ssamp = &bs->ssamp;
152         ShadeResult shr;
153         VlakRen *vlr = shi->vlr;
154
155         shade_input_init_material(shi);
156
157         if (bs->type == RE_BAKE_AO) {
158                 ambient_occlusion(shi);
159
160                 if (R.r.bake_flag & R_BAKE_NORMALIZE) {
161                         copy_v3_v3(shr.combined, shi->ao);
162                 }
163                 else {
164                         zero_v3(shr.combined);
165                         environment_lighting_apply(shi, &shr);
166                 }
167         }
168         else {
169                 if (bs->type == RE_BAKE_SHADOW) /* Why do shadows set the color anyhow?, ignore material color for baking */
170                         shi->r = shi->g = shi->b = 1.0f;
171
172                 shade_input_set_shade_texco(shi);
173                 
174                 /* only do AO for a full bake (and obviously AO bakes)
175                  * AO for light bakes is a leftover and might not be needed */
176                 if (ELEM3(bs->type, RE_BAKE_ALL, RE_BAKE_AO, RE_BAKE_LIGHT))
177                         shade_samples_do_AO(ssamp);
178                 
179                 if (shi->mat->nodetree && shi->mat->use_nodes) {
180                         ntreeShaderExecTree(shi->mat->nodetree, shi, &shr);
181                         shi->mat = vlr->mat;  /* shi->mat is being set in nodetree */
182                 }
183                 else
184                         shade_material_loop(shi, &shr);
185
186                 if (bs->type == RE_BAKE_NORMALS) {
187                         float nor[3];
188
189                         copy_v3_v3(nor, shi->vn);
190
191                         if (R.r.bake_normal_space == R_BAKE_SPACE_CAMERA) {
192                                 /* pass */
193                         }
194                         else if (R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
195                                 float mat[3][3], imat[3][3];
196
197                                 /* bitangent */
198                                 if (tvn && ttang) {
199                                         copy_v3_v3(mat[0], ttang);
200                                         cross_v3_v3v3(mat[1], tvn, ttang);
201                                         mul_v3_fl(mat[1], ttang[3]);
202                                         copy_v3_v3(mat[2], tvn);
203                                 }
204                                 else {
205                                         copy_v3_v3(mat[0], shi->nmaptang);
206                                         cross_v3_v3v3(mat[1], shi->nmapnorm, shi->nmaptang);
207                                         mul_v3_fl(mat[1], shi->nmaptang[3]);
208                                         copy_v3_v3(mat[2], shi->nmapnorm);
209                                 }
210
211                                 invert_m3_m3(imat, mat);
212                                 mul_m3_v3(imat, nor);
213                         }
214                         else if (R.r.bake_normal_space == R_BAKE_SPACE_OBJECT)
215                                 mul_mat3_m4_v3(ob->imat_ren, nor);  /* ob->imat_ren includes viewinv! */
216                         else if (R.r.bake_normal_space == R_BAKE_SPACE_WORLD)
217                                 mul_mat3_m4_v3(R.viewinv, nor);
218
219                         normalize_v3(nor); /* in case object has scaling */
220
221                         /* The invert of the red channel is to make
222                          * the normal map compliant with the outside world.
223                          * It needs to be done because in Blender
224                          * the normal used in the renderer points inward. It is generated
225                          * this way in calc_vertexnormals(). Should this ever change
226                          * this negate must be removed. */
227                         shr.combined[0] = (-nor[0]) / 2.0f + 0.5f;
228                         shr.combined[1] = nor[1]    / 2.0f + 0.5f;
229                         shr.combined[2] = nor[2]    / 2.0f + 0.5f;
230                 }
231                 else if (bs->type == RE_BAKE_TEXTURE) {
232                         copy_v3_v3(shr.combined, &shi->r);
233                         shr.alpha = shi->alpha;
234                 }
235                 else if (bs->type == RE_BAKE_SHADOW) {
236                         copy_v3_v3(shr.combined, shr.shad);
237                         shr.alpha = shi->alpha;
238                 }
239                 else if (bs->type == RE_BAKE_SPEC_COLOR) {
240                         copy_v3_v3(shr.combined, &shi->specr);
241                         shr.alpha = 1.0f;
242                 }
243                 else if (bs->type == RE_BAKE_SPEC_INTENSITY) {
244                         copy_v3_fl(shr.combined, shi->spec);
245                         shr.alpha = 1.0f;
246                 }
247                 else if (bs->type == RE_BAKE_MIRROR_COLOR) {
248                         copy_v3_v3(shr.combined, &shi->mirr);
249                         shr.alpha = 1.0f;
250                 }
251                 else if (bs->type == RE_BAKE_MIRROR_INTENSITY) {
252                         copy_v3_fl(shr.combined, shi->ray_mirror);
253                         shr.alpha = 1.0f;
254                 }
255                 else if (bs->type == RE_BAKE_ALPHA) {
256                         copy_v3_fl(shr.combined, shi->alpha);
257                         shr.alpha = 1.0f;
258                 }
259                 else if (bs->type == RE_BAKE_EMIT) {
260                         copy_v3_fl(shr.combined, shi->emit);
261                         shr.alpha = 1.0f;
262                 }
263                 else if (bs->type == RE_BAKE_VERTEX_COLORS) {
264                         copy_v3_v3(shr.combined, shi->vcol);
265                         shr.alpha = 1.0;
266                 }
267         }
268         
269         if (bs->rect_float && !bs->vcol) {
270                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
271                 copy_v3_v3(col, shr.combined);
272                 if (bs->type == RE_BAKE_ALL || bs->type == RE_BAKE_TEXTURE) {
273                         col[3] = shr.alpha;
274                 }
275                 else {
276                         col[3] = 1.0;
277                 }
278         }
279         else {
280                 /* Target is char (LDR). */
281                 unsigned char col[4];
282
283                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
284                         float rgb[3];
285
286                         copy_v3_v3(rgb, shr.combined);
287                         if (R.scene_color_manage) {
288                                 /* Vertex colors have no way to specify color space, so they
289                                  * default to sRGB. */
290                                 if (!bs->vcol)
291                                         IMB_colormanagement_scene_linear_to_colorspace_v3(rgb, bs->rect_colorspace);
292                                 else
293                                         linearrgb_to_srgb_v3_v3(rgb, rgb);
294                         }
295                         rgb_float_to_uchar(col, rgb);
296                 }
297                 else {
298                         rgb_float_to_uchar(col, shr.combined);
299                 }
300                 
301                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
302                         col[3] = FTOCHAR(shr.alpha);
303                 }
304                 else {
305                         col[3] = 255;
306                 }
307
308                 if (bs->vcol) {
309                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
310                          * but is used only for vertex painting). */
311                         bs->vcol->r = col[0];
312                         bs->vcol->g = col[1];
313                         bs->vcol->b = col[2];
314                 }
315                 else {
316                         unsigned char *imcol = (unsigned char *)(bs->rect + bs->rectx * y + x);
317                         copy_v4_v4_char((char *)imcol, (char *)col);
318                 }
319
320         }
321         
322         if (bs->rect_mask) {
323                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
324         }
325 }
326
327 static void bake_displacement(void *handle, ShadeInput *UNUSED(shi), float dist, int x, int y)
328 {
329         BakeShade *bs = handle;
330         float disp;
331
332         if (R.r.bake_flag & R_BAKE_NORMALIZE) {
333                 if (R.r.bake_maxdist)
334                         disp = (dist + R.r.bake_maxdist) / (R.r.bake_maxdist * 2);  /* alter the range from [-bake_maxdist, bake_maxdist] to [0, 1]*/
335                 else
336                         disp = dist;
337         }
338         else {
339                 disp = 0.5f + dist; /* alter the range from [-0.5,0.5] to [0,1]*/
340         }
341
342         if (bs->displacement_buffer) {
343                 float *displacement = bs->displacement_buffer + (bs->rectx * y + x);
344                 *displacement = disp;
345                 bs->displacement_min = min_ff(bs->displacement_min, disp);
346                 bs->displacement_max = max_ff(bs->displacement_max, disp);
347         }
348
349         if (bs->rect_float && !bs->vcol) {
350                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
351                 col[0] = col[1] = col[2] = disp;
352                 col[3] = 1.0f;
353         }
354         else {
355                 /* Target is char (LDR). */
356                 unsigned char col[4];
357                 col[0] = col[1] = col[2] = FTOCHAR(disp);
358                 col[3] = 255;
359
360                 if (bs->vcol) {
361                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
362                          * but is used only for vertex painting). */
363                         bs->vcol->r = col[0];
364                         bs->vcol->g = col[1];
365                         bs->vcol->b = col[2];
366                 }
367                 else {
368                         char *imcol = (char *)(bs->rect + bs->rectx * y + x);
369                         copy_v4_v4_char((char *)imcol, (char *)col);
370                 }
371         }
372         if (bs->rect_mask) {
373                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
374         }
375 }
376
377 static int bake_intersect_tree(RayObject *raytree, Isect *isect, float *start, float *dir, float sign, float *hitco, float *dist)
378 {
379         float maxdist;
380         int hit;
381
382         /* might be useful to make a user setting for maxsize*/
383         if (R.r.bake_maxdist > 0.0f)
384                 maxdist = R.r.bake_maxdist;
385         else
386                 maxdist = RE_RAYTRACE_MAXDIST + R.r.bake_biasdist;
387
388         /* 'dir' is always normalized */
389         madd_v3_v3v3fl(isect->start, start, dir, -R.r.bake_biasdist);
390
391         mul_v3_v3fl(isect->dir, dir, sign);
392
393         isect->dist = maxdist;
394
395         hit = RE_rayobject_raycast(raytree, isect);
396         if (hit) {
397                 madd_v3_v3v3fl(hitco, isect->start, isect->dir, isect->dist);
398
399                 *dist = isect->dist;
400         }
401
402         return hit;
403 }
404
405 static void bake_set_vlr_dxyco(BakeShade *bs, float *uv1, float *uv2, float *uv3)
406 {
407         VlakRen *vlr = bs->vlr;
408         float A, d1, d2, d3, *v1, *v2, *v3;
409
410         if (bs->quad) {
411                 v1 = vlr->v1->co;
412                 v2 = vlr->v3->co;
413                 v3 = vlr->v4->co;
414         }
415         else {
416                 v1 = vlr->v1->co;
417                 v2 = vlr->v2->co;
418                 v3 = vlr->v3->co;
419         }
420
421         /* formula derived from barycentric coordinates:
422          * (uvArea1*v1 + uvArea2*v2 + uvArea3*v3)/uvArea
423          * then taking u and v partial derivatives to get dxco and dyco */
424         A = (uv2[0] - uv1[0]) * (uv3[1] - uv1[1]) - (uv3[0] - uv1[0]) * (uv2[1] - uv1[1]);
425
426         if (fabsf(A) > FLT_EPSILON) {
427                 A = 0.5f / A;
428
429                 d1 = uv2[1] - uv3[1];
430                 d2 = uv3[1] - uv1[1];
431                 d3 = uv1[1] - uv2[1];
432                 bs->dxco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
433                 bs->dxco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
434                 bs->dxco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
435
436                 d1 = uv3[0] - uv2[0];
437                 d2 = uv1[0] - uv3[0];
438                 d3 = uv2[0] - uv1[0];
439                 bs->dyco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
440                 bs->dyco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
441                 bs->dyco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
442         }
443         else {
444                 bs->dxco[0] = bs->dxco[1] = bs->dxco[2] = 0.0f;
445                 bs->dyco[0] = bs->dyco[1] = bs->dyco[2] = 0.0f;
446         }
447
448         if (bs->obi->flag & R_TRANSFORMED) {
449                 mul_m3_v3(bs->obi->nmat, bs->dxco);
450                 mul_m3_v3(bs->obi->nmat, bs->dyco);
451         }
452 }
453
454 static void do_bake_shade(void *handle, int x, int y, float u, float v)
455 {
456         BakeShade *bs = handle;
457         VlakRen *vlr = bs->vlr;
458         ObjectInstanceRen *obi = bs->obi;
459         Object *ob = obi->obr->ob;
460         float l, *v1, *v2, *v3, tvn[3], ttang[4];
461         int quad;
462         ShadeSample *ssamp = &bs->ssamp;
463         ShadeInput *shi = ssamp->shi;
464
465         /* fast threadsafe break test */
466         if (R.test_break(R.tbh))
467                 return;
468
469         /* setup render coordinates */
470         if (bs->quad) {
471                 v1 = vlr->v1->co;
472                 v2 = vlr->v3->co;
473                 v3 = vlr->v4->co;
474         }
475         else {
476                 v1 = vlr->v1->co;
477                 v2 = vlr->v2->co;
478                 v3 = vlr->v3->co;
479         }
480
481         l = 1.0f - u - v;
482
483         /* shrink barycentric coordinates inwards slightly to avoid some issues
484          * where baking selected to active might just miss the other face at the
485          * near the edge of a face */
486         if (bs->actob) {
487                 const float eps = 1.0f - 1e-4f;
488                 float invsum;
489
490                 u = (u - 0.5f) * eps + 0.5f;
491                 v = (v - 0.5f) * eps + 0.5f;
492                 l = (l - 0.5f) * eps + 0.5f;
493
494                 invsum = 1.0f / (u + v + l);
495
496                 u *= invsum;
497                 v *= invsum;
498                 l *= invsum;
499         }
500
501         /* renderco */
502         shi->co[0] = l * v3[0] + u * v1[0] + v * v2[0];
503         shi->co[1] = l * v3[1] + u * v1[1] + v * v2[1];
504         shi->co[2] = l * v3[2] + u * v1[2] + v * v2[2];
505
506         /* avoid self shadow with vertex bake from adjacent faces [#33729] */
507         if ((bs->vcol != NULL) && (bs->actob == NULL)) {
508                 madd_v3_v3fl(shi->co, vlr->n, 0.0001f);
509         }
510
511         if (obi->flag & R_TRANSFORMED)
512                 mul_m4_v3(obi->mat, shi->co);
513
514         copy_v3_v3(shi->dxco, bs->dxco);
515         copy_v3_v3(shi->dyco, bs->dyco);
516
517         quad = bs->quad;
518         bake_set_shade_input(obi, vlr, shi, quad, 0, x, y, u, v);
519
520         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
521                 shade_input_set_shade_texco(shi);
522                 copy_v3_v3(tvn, shi->nmapnorm);
523                 copy_v4_v4(ttang, shi->nmaptang);
524         }
525
526         /* if we are doing selected to active baking, find point on other face */
527         if (bs->actob) {
528                 Isect isec, minisec;
529                 float co[3], minco[3], dist, mindist = 0.0f;
530                 int hit, sign, dir = 1;
531
532                 /* intersect with ray going forward and backward*/
533                 hit = 0;
534                 memset(&minisec, 0, sizeof(minisec));
535                 minco[0] = minco[1] = minco[2] = 0.0f;
536
537                 copy_v3_v3(bs->dir, shi->vn);
538
539                 for (sign = -1; sign <= 1; sign += 2) {
540                         memset(&isec, 0, sizeof(isec));
541                         isec.mode = RE_RAY_MIRROR;
542
543                         isec.orig.ob   = obi;
544                         isec.orig.face = vlr;
545                         isec.userdata = bs->actob;
546                         isec.check = RE_CHECK_VLR_BAKE;
547                         isec.skip = RE_SKIP_VLR_NEIGHBOUR;
548
549                         if (bake_intersect_tree(R.raytree, &isec, shi->co, shi->vn, sign, co, &dist)) {
550                                 if (!hit || len_squared_v3v3(shi->co, co) < len_squared_v3v3(shi->co, minco)) {
551                                         minisec = isec;
552                                         mindist = dist;
553                                         copy_v3_v3(minco, co);
554                                         hit = 1;
555                                         dir = sign;
556                                 }
557                         }
558                 }
559
560                 if (ELEM(bs->type, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE)) {
561                         if (hit)
562                                 bake_displacement(handle, shi, (dir == -1) ? mindist : -mindist, x, y);
563                         else
564                                 bake_displacement(handle, shi, 0.0f, x, y);
565                         return;
566                 }
567
568                 /* if hit, we shade from the new point, otherwise from point one starting face */
569                 if (hit) {
570                         obi = (ObjectInstanceRen *)minisec.hit.ob;
571                         vlr = (VlakRen *)minisec.hit.face;
572                         quad = (minisec.isect == 2);
573                         copy_v3_v3(shi->co, minco);
574
575                         u = -minisec.u;
576                         v = -minisec.v;
577                         bake_set_shade_input(obi, vlr, shi, quad, 1, x, y, u, v);
578                 }
579         }
580
581         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT)
582                 bake_shade(handle, ob, shi, quad, x, y, u, v, tvn, ttang);
583         else
584                 bake_shade(handle, ob, shi, quad, x, y, u, v, NULL, NULL);
585 }
586
587 static int get_next_bake_face(BakeShade *bs)
588 {
589         ObjectRen *obr;
590         VlakRen *vlr;
591         MTFace *tface;
592         static int v = 0, vdone = false;
593         static ObjectInstanceRen *obi = NULL;
594
595         if (bs == NULL) {
596                 vlr = NULL;
597                 v = vdone = false;
598                 obi = R.instancetable.first;
599                 return 0;
600         }
601         
602         BLI_lock_thread(LOCK_CUSTOM1);
603
604         for (; obi; obi = obi->next, v = 0) {
605                 obr = obi->obr;
606
607                 for (; v < obr->totvlak; v++) {
608                         vlr = RE_findOrAddVlak(obr, v);
609
610                         if ((bs->actob && bs->actob == obr->ob) || (!bs->actob && (obr->ob->flag & SELECT))) {
611                                 if (R.r.bake_flag & R_BAKE_VCOL) {
612                                         /* Gather face data for vertex color bake */
613                                         Mesh *me;
614                                         int *origindex, vcollayer;
615                                         CustomDataLayer *cdl;
616
617                                         if (obr->ob->type != OB_MESH)
618                                                 continue;
619                                         me = obr->ob->data;
620
621                                         origindex = RE_vlakren_get_origindex(obr, vlr, 0);
622                                         if (origindex == NULL)
623                                                 continue;
624                                         if (*origindex >= me->totpoly) {
625                                                 /* Small hack for Array modifier, which gives false
626                                                  * original indices - z0r */
627                                                 continue;
628                                         }
629 #if 0
630                                         /* Only shade selected faces. */
631                                         if ((me->mface[*origindex].flag & ME_FACE_SEL) == 0)
632                                                 continue;
633 #endif
634
635                                         vcollayer = CustomData_get_render_layer_index(&me->ldata, CD_MLOOPCOL);
636                                         if (vcollayer == -1)
637                                                 continue;
638
639                                         cdl = &me->ldata.layers[vcollayer];
640                                         bs->mpoly = me->mpoly + *origindex;
641                                         bs->vcol = ((MLoopCol *)cdl->data) + bs->mpoly->loopstart;
642                                         bs->mloop = me->mloop + bs->mpoly->loopstart;
643
644                                         /* Tag mesh for reevaluation. */
645                                         me->id.flag |= LIB_DOIT;
646                                 }
647                                 else {
648                                         Image *ima = NULL;
649                                         ImBuf *ibuf = NULL;
650                                         const float vec_alpha[4] = {0.0f, 0.0f, 0.0f, 0.0f};
651                                         const float vec_solid[4] = {0.0f, 0.0f, 0.0f, 1.0f};
652                                         const float nor_alpha[4] = {0.5f, 0.5f, 1.0f, 0.0f};
653                                         const float nor_solid[4] = {0.5f, 0.5f, 1.0f, 1.0f};
654                                         const float disp_alpha[4] = {0.5f, 0.5f, 0.5f, 0.0f};
655                                         const float disp_solid[4] = {0.5f, 0.5f, 0.5f, 1.0f};
656
657                                         tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
658
659                                         if (!tface || !tface->tpage)
660                                                 continue;
661
662                                         ima = tface->tpage;
663                                         ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
664
665                                         if (ibuf == NULL)
666                                                 continue;
667
668                                         if (ibuf->rect == NULL && ibuf->rect_float == NULL) {
669                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
670                                                 continue;
671                                         }
672
673                                         if (ibuf->rect_float && !(ibuf->channels == 0 || ibuf->channels == 4)) {
674                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
675                                                 continue;
676                                         }
677                                         
678                                         if (ima->flag & IMA_USED_FOR_RENDER) {
679                                                 ima->id.flag &= ~LIB_DOIT;
680                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
681                                                 continue;
682                                         }
683                                         
684                                         /* find the image for the first time? */
685                                         if (ima->id.flag & LIB_DOIT) {
686                                                 ima->id.flag &= ~LIB_DOIT;
687                                                 
688                                                 /* we either fill in float or char, this ensures things go fine */
689                                                 if (ibuf->rect_float)
690                                                         imb_freerectImBuf(ibuf);
691                                                 /* clear image */
692                                                 if (R.r.bake_flag & R_BAKE_CLEAR) {
693                                                         if (R.r.bake_mode == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT)
694                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? nor_alpha : nor_solid);
695                                                         else if (ELEM(R.r.bake_mode, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE))
696                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? disp_alpha : disp_solid);
697                                                         else
698                                                                 IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? vec_alpha : vec_solid);
699                                                 }
700                                                 /* might be read by UI to set active image for display */
701                                                 R.bakebuf = ima;
702                                         }
703
704                                         /* Tag image for redraw. */
705                                         ibuf->userflags |= IB_DISPLAY_BUFFER_INVALID;
706                                         BKE_image_release_ibuf(ima, ibuf, NULL);
707                                 }
708
709                                 bs->obi = obi;
710                                 bs->vlr = vlr;
711                                 bs->vdone++;  /* only for error message if nothing was rendered */
712                                 v++;
713                                 BLI_unlock_thread(LOCK_CUSTOM1);
714                                 return 1;
715                         }
716                 }
717         }
718         
719         BLI_unlock_thread(LOCK_CUSTOM1);
720         return 0;
721 }
722
723 static void bake_single_vertex(BakeShade *bs, VertRen *vert, float u, float v)
724 {
725         int *origindex, i;
726         MLoopCol *basevcol;
727         MLoop *mloop;
728
729         origindex = RE_vertren_get_origindex(bs->obi->obr, vert, 0);
730         if (!origindex || *origindex == ORIGINDEX_NONE)
731                 return;
732
733         /* Search for matching vertex index and apply shading. */
734         for (i = 0; i < bs->mpoly->totloop; i++) {
735                 mloop = bs->mloop + i;
736                 if (mloop->v != *origindex)
737                         continue;
738                 basevcol = bs->vcol;
739                 bs->vcol = basevcol + i;
740                 do_bake_shade(bs, 0, 0, u, v);
741                 bs->vcol = basevcol;
742                 break;
743         }
744 }
745
746 /* Bake all vertices of a face. Actually, this still works on a face-by-face
747  * basis, and each vertex on each face is shaded. Vertex colors are a property
748  * of loops, not vertices. */
749 static void shade_verts(BakeShade *bs)
750 {
751         VlakRen *vlr = bs->vlr;
752
753         /* Disable baking to image; write to vcol instead. vcol pointer is set in
754          * bake_single_vertex. */
755         bs->ima = NULL;
756         bs->rect = NULL;
757         bs->rect_float = NULL;
758         bs->displacement_buffer = NULL;
759         bs->displacement_min = FLT_MAX;
760         bs->displacement_max = -FLT_MAX;
761
762         bs->quad = 0;
763
764         /* No anti-aliasing for vertices. */
765         zero_v3(bs->dxco);
766         zero_v3(bs->dyco);
767
768         /* Shade each vertex of the face. u and v are barycentric coordinates; since
769          * we're only interested in vertices, these will be 0 or 1. */
770         if ((vlr->flag & R_FACE_SPLIT) == 0) {
771                 /* Processing triangle face, whole quad, or first half of split quad. */
772
773                 bake_single_vertex(bs, bs->vlr->v1, 1.0f, 0.0f);
774                 bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
775                 bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
776
777                 if (vlr->v4) {
778                         bs->quad = 1;
779                         bake_single_vertex(bs, bs->vlr->v4, 0.0f, 0.0f);
780                 }
781         }
782         else {
783                 /* Processing second half of split quad. Only one vertex to go. */
784                 if (vlr->flag & R_DIVIDE_24) {
785                         bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
786                 }
787                 else {
788                         bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
789                 }
790         }
791 }
792
793 /* already have tested for tface and ima and zspan */
794 static void shade_tface(BakeShade *bs)
795 {
796         VlakRen *vlr = bs->vlr;
797         ObjectInstanceRen *obi = bs->obi;
798         ObjectRen *obr = obi->obr;
799         MTFace *tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
800         Image *ima = tface->tpage;
801         float vec[4][2];
802         int a, i1, i2, i3;
803         
804         /* check valid zspan */
805         if (ima != bs->ima) {
806                 BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
807
808                 bs->ima = ima;
809                 bs->ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
810                 /* note, these calls only free/fill contents of zspan struct, not zspan itself */
811                 zbuf_free_span(bs->zspan);
812                 zbuf_alloc_span(bs->zspan, bs->ibuf->x, bs->ibuf->y, R.clipcrop);
813         }
814
815         bs->rectx = bs->ibuf->x;
816         bs->recty = bs->ibuf->y;
817         bs->rect = bs->ibuf->rect;
818         bs->rect_colorspace = bs->ibuf->rect_colorspace;
819         bs->rect_float = bs->ibuf->rect_float;
820         bs->vcol = NULL;
821         bs->quad = 0;
822         bs->rect_mask = NULL;
823         bs->displacement_buffer = NULL;
824
825         if (bs->use_mask || bs->use_displacement_buffer) {
826                 BakeImBufuserData *userdata = bs->ibuf->userdata;
827                 if (userdata == NULL) {
828                         BLI_lock_thread(LOCK_CUSTOM1);
829                         userdata = bs->ibuf->userdata;
830                         if (userdata == NULL) /* since the thread was locked, its possible another thread alloced the value */
831                                 userdata = MEM_callocN(sizeof(BakeImBufuserData), "BakeImBufuserData");
832
833                         if (bs->use_mask) {
834                                 if (userdata->mask_buffer == NULL) {
835                                         userdata->mask_buffer = MEM_callocN(sizeof(char) * bs->rectx * bs->recty, "BakeMask");
836                                 }
837                         }
838
839                         if (bs->use_displacement_buffer) {
840                                 if (userdata->displacement_buffer == NULL) {
841                                         userdata->displacement_buffer = MEM_callocN(sizeof(float) * bs->rectx * bs->recty, "BakeDisp");
842                                 }
843                         }
844
845                         bs->ibuf->userdata = userdata;
846
847                         BLI_unlock_thread(LOCK_CUSTOM1);
848                 }
849
850                 bs->rect_mask = userdata->mask_buffer;
851                 bs->displacement_buffer = userdata->displacement_buffer;
852         }
853         
854         /* get pixel level vertex coordinates */
855         for (a = 0; a < 4; a++) {
856                 /* Note, workaround for pixel aligned UVs which are common and can screw up our intersection tests
857                  * where a pixel gets in between 2 faces or the middle of a quad,
858                  * camera aligned quads also have this problem but they are less common.
859                  * Add a small offset to the UVs, fixes bug #18685 - Campbell */
860                 vec[a][0] = tface->uv[a][0] * (float)bs->rectx - (0.5f + 0.001f);
861                 vec[a][1] = tface->uv[a][1] * (float)bs->recty - (0.5f + 0.002f);
862         }
863
864         /* UV indices have to be corrected for possible quad->tria splits */
865         i1 = 0; i2 = 1; i3 = 2;
866         vlr_set_uv_indices(vlr, &i1, &i2, &i3);
867         bake_set_vlr_dxyco(bs, vec[i1], vec[i2], vec[i3]);
868         zspan_scanconvert(bs->zspan, bs, vec[i1], vec[i2], vec[i3], do_bake_shade);
869         
870         if (vlr->v4) {
871                 bs->quad = 1;
872                 bake_set_vlr_dxyco(bs, vec[0], vec[2], vec[3]);
873                 zspan_scanconvert(bs->zspan, bs, vec[0], vec[2], vec[3], do_bake_shade);
874         }
875 }
876
877 static void *do_bake_thread(void *bs_v)
878 {
879         BakeShade *bs = bs_v;
880
881         while (get_next_bake_face(bs)) {
882                 if (R.r.bake_flag & R_BAKE_VCOL) {
883                         shade_verts(bs);
884                 }
885                 else {
886                         shade_tface(bs);
887                 }
888                 
889                 /* fast threadsafe break test */
890                 if (R.test_break(R.tbh))
891                         break;
892
893                 /* access is not threadsafe but since its just true/false probably ok
894                  * only used for interactive baking */
895                 if (bs->do_update) {
896                         *bs->do_update = true;
897                 }
898         }
899         bs->ready = true;
900
901         BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
902
903         return NULL;
904 }
905
906 void RE_bake_ibuf_filter(ImBuf *ibuf, char *mask, const int filter)
907 {
908         /* must check before filtering */
909         const short is_new_alpha = (ibuf->planes != R_IMF_PLANES_RGBA) && BKE_imbuf_alpha_test(ibuf);
910
911         /* Margin */
912         if (filter) {
913                 IMB_filter_extend(ibuf, mask, filter);
914         }
915
916         /* if the bake results in new alpha then change the image setting */
917         if (is_new_alpha) {
918                 ibuf->planes = R_IMF_PLANES_RGBA;
919         }
920         else {
921                 if (filter && ibuf->planes != R_IMF_PLANES_RGBA) {
922                         /* clear alpha added by filtering */
923                         IMB_rectfill_alpha(ibuf, 1.0f);
924                 }
925         }
926 }
927
928 void RE_bake_ibuf_normalize_displacement(ImBuf *ibuf, float *displacement, char *mask, float displacement_min, float displacement_max)
929 {
930         int i;
931         float *current_displacement = displacement;
932         char *current_mask = mask;
933         float max_distance;
934
935         max_distance = max_ff(fabsf(displacement_min), fabsf(displacement_max));
936
937         for (i = 0; i < ibuf->x * ibuf->y; i++) {
938                 if (*current_mask == FILTER_MASK_USED) {
939                         float normalized_displacement;
940
941                         if (max_distance > 1e-5f)
942                                 normalized_displacement = (*current_displacement + max_distance) / (max_distance * 2);
943                         else
944                                 normalized_displacement = 0.5f;
945
946                         if (ibuf->rect_float) {
947                                 /* currently baking happens to RGBA only */
948                                 float *fp = ibuf->rect_float + i * 4;
949                                 fp[0] = fp[1] = fp[2] = normalized_displacement;
950                                 fp[3] = 1.0f;
951                         }
952
953                         if (ibuf->rect) {
954                                 unsigned char *cp = (unsigned char *) (ibuf->rect + i);
955                                 cp[0] = cp[1] = cp[2] = FTOCHAR(normalized_displacement);
956                                 cp[3] = 255;
957                         }
958                 }
959
960                 current_displacement++;
961                 current_mask++;
962         }
963 }
964
965 /* using object selection tags, the faces with UV maps get baked */
966 /* render should have been setup */
967 /* returns 0 if nothing was handled */
968 int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_update, float *progress)
969 {
970         BakeShade *handles;
971         ListBase threads;
972         Image *ima;
973         int a, vdone = false, result = BAKE_RESULT_OK;
974         bool use_mask = false;
975         bool use_displacement_buffer = false;
976         
977         re->scene_color_manage = BKE_scene_check_color_management_enabled(re->scene);
978         
979         /* initialize render global */
980         R = *re;
981         R.bakebuf = NULL;
982
983         /* initialize static vars */
984         get_next_bake_face(NULL);
985         
986         /* do we need a mask? */
987         if (re->r.bake_filter)
988                 use_mask = true;
989
990         /* do we need buffer to store displacements  */
991         if (ELEM(type, RE_BAKE_DISPLACEMENT, RE_BAKE_DERIVATIVE)) {
992                 if (((R.r.bake_flag & R_BAKE_NORMALIZE) && R.r.bake_maxdist == 0.0f) ||
993                     (type == RE_BAKE_DERIVATIVE))
994                 {
995                         use_displacement_buffer = true;
996                         use_mask = true;
997                 }
998         }
999
1000         /* baker uses this flag to detect if image was initialized */
1001         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
1002                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
1003                         ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
1004                         ima->id.flag |= LIB_DOIT;
1005                         ima->flag &= ~IMA_USED_FOR_RENDER;
1006                         if (ibuf) {
1007                                 ibuf->userdata = NULL; /* use for masking if needed */
1008                         }
1009                         BKE_image_release_ibuf(ima, ibuf, NULL);
1010                 }
1011         }
1012
1013         if (R.r.bake_flag & R_BAKE_VCOL) {
1014                 /* untag all meshes */
1015                 tag_main_lb(&G.main->mesh, false);
1016         }
1017
1018         BLI_init_threads(&threads, do_bake_thread, re->r.threads);
1019
1020         handles = MEM_callocN(sizeof(BakeShade) * re->r.threads, "BakeShade");
1021
1022         /* get the threads running */
1023         for (a = 0; a < re->r.threads; a++) {
1024                 /* set defaults in handles */
1025                 handles[a].ssamp.shi[0].lay = re->lay;
1026
1027                 if (type == RE_BAKE_SHADOW) {
1028                         handles[a].ssamp.shi[0].passflag = SCE_PASS_SHADOW;
1029                 }
1030                 else {
1031                         handles[a].ssamp.shi[0].passflag = SCE_PASS_COMBINED;
1032                 }
1033                 handles[a].ssamp.shi[0].combinedflag = ~(SCE_PASS_SPEC);
1034                 handles[a].ssamp.shi[0].thread = a;
1035                 handles[a].ssamp.tot = 1;
1036
1037                 handles[a].type = type;
1038                 handles[a].actob = actob;
1039                 if (R.r.bake_flag & R_BAKE_VCOL)
1040                         handles[a].zspan = NULL;
1041                 else
1042                         handles[a].zspan = MEM_callocN(sizeof(ZSpan), "zspan for bake");
1043                 
1044                 handles[a].use_mask = use_mask;
1045                 handles[a].use_displacement_buffer = use_displacement_buffer;
1046
1047                 handles[a].do_update = do_update; /* use to tell the view to update */
1048                 
1049                 handles[a].displacement_min = FLT_MAX;
1050                 handles[a].displacement_max = -FLT_MAX;
1051
1052                 BLI_insert_thread(&threads, &handles[a]);
1053         }
1054         
1055         /* wait for everything to be done */
1056         a = 0;
1057         while (a != re->r.threads) {
1058                 PIL_sleep_ms(50);
1059
1060                 /* calculate progress */
1061                 for (vdone = false, a = 0; a < re->r.threads; a++)
1062                         vdone += handles[a].vdone;
1063                 if (progress)
1064                         *progress = (float)(vdone / (float)re->totvlak);
1065
1066                 for (a = 0; a < re->r.threads; a++) {
1067                         if (handles[a].ready == false) {
1068                                 break;
1069                         }
1070                 }
1071         }
1072
1073         /* filter and refresh images */
1074         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
1075                 float displacement_min = FLT_MAX, displacement_max = -FLT_MAX;
1076
1077                 if (use_displacement_buffer) {
1078                         for (a = 0; a < re->r.threads; a++) {
1079                                 displacement_min = min_ff(displacement_min, handles[a].displacement_min);
1080                                 displacement_max = max_ff(displacement_max, handles[a].displacement_max);
1081                         }
1082                 }
1083
1084                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
1085                         if ((ima->id.flag & LIB_DOIT) == 0) {
1086                                 ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
1087                                 BakeImBufuserData *userdata;
1088
1089                                 if (ima->flag & IMA_USED_FOR_RENDER)
1090                                         result = BAKE_RESULT_FEEDBACK_LOOP;
1091
1092                                 if (!ibuf)
1093                                         continue;
1094
1095                                 userdata = (BakeImBufuserData *)ibuf->userdata;
1096                                 if (userdata) {
1097                                         if (use_displacement_buffer) {
1098                                                 if (type == RE_BAKE_DERIVATIVE) {
1099                                                         float user_scale = (R.r.bake_flag & R_BAKE_USERSCALE) ? R.r.bake_user_scale : -1.0f;
1100                                                         RE_bake_make_derivative(ibuf, userdata->displacement_buffer, userdata->mask_buffer,
1101                                                                                 displacement_min, displacement_max, user_scale);
1102                                                 }
1103                                                 else {
1104                                                         RE_bake_ibuf_normalize_displacement(ibuf, userdata->displacement_buffer, userdata->mask_buffer,
1105                                                                                             displacement_min, displacement_max);
1106                                                 }
1107                                         }
1108
1109                                         RE_bake_ibuf_filter(ibuf, userdata->mask_buffer, re->r.bake_filter);
1110                                 }
1111
1112                                 ibuf->userflags |= IB_BITMAPDIRTY;
1113                                 BKE_image_release_ibuf(ima, ibuf, NULL);
1114                         }
1115                 }
1116
1117                 /* calculate return value */
1118                 for (a = 0; a < re->r.threads; a++) {
1119                         zbuf_free_span(handles[a].zspan);
1120                         MEM_freeN(handles[a].zspan);
1121                 }
1122         }
1123
1124         MEM_freeN(handles);
1125         
1126         BLI_end_threads(&threads);
1127
1128         if (vdone == 0) {
1129                 result = BAKE_RESULT_NO_OBJECTS;
1130         }
1131
1132         return result;
1133 }
1134
1135 struct Image *RE_bake_shade_get_image(void)
1136 {
1137         return R.bakebuf;
1138 }
1139
1140 /* **************** Derivative Maps Baker **************** */
1141
1142 static void add_single_heights_margin(const ImBuf *ibuf, const char *mask, float *heights_buffer)
1143 {
1144         int x ,y;
1145
1146         for (y = 0; y < ibuf->y; y++) {
1147                 for (x = 0; x < ibuf->x; x++) {
1148                         int index = ibuf->x * y + x;
1149
1150                         /* If unassigned pixel, look for neighbors. */
1151                         if (mask[index] != FILTER_MASK_USED) {
1152                                 float height_acc = 0;
1153                                 int denom = 0;
1154                                 int i, j;
1155
1156                                 for (j = -1; j <= 1; j++)
1157                                         for (i = -1; i <= 1; i++) {
1158                                                 int w = (i == 0 ? 1 : 0) + (j == 0 ? 1 : 0) + 1;
1159
1160                                                 if (i != 0 || j != 0) {
1161                                                         int index2 = 0;
1162                                                         int x0 = x + i;
1163                                                         int y0 = y + j;
1164
1165                                                         CLAMP(x0, 0, ibuf->x - 1);
1166                                                         CLAMP(y0, 0, ibuf->y - 1);
1167
1168                                                         index2 = ibuf->x * y0 + x0;
1169
1170                                                         if (mask[index2] == FILTER_MASK_USED) {
1171                                                                 height_acc += w * heights_buffer[index2];
1172                                                                 denom += w;
1173                                                         }
1174                                                 }
1175                                         }
1176
1177                                 /* Insert final value. */
1178                                 if (denom > 0) {
1179                                         heights_buffer[index] = height_acc / denom;
1180                                 }
1181                         }
1182                 }
1183         }
1184 }
1185
1186 /* returns user-scale */
1187 float RE_bake_make_derivative(ImBuf *ibuf, float *heights_buffer, const char *mask,
1188                               const float height_min, const float height_max,
1189                               const float fmult)
1190 {
1191         const float delta_height = height_max - height_min;
1192         const float denom = delta_height > 0.0f ? (8 * delta_height) : 1.0f;
1193         bool auto_range_fit = fmult <= 0.0f;
1194         float max_num_deriv = -1.0f;
1195         int x, y, index;
1196
1197         /* Need a single margin to calculate good derivatives. */
1198         add_single_heights_margin(ibuf, mask, heights_buffer);
1199
1200         if (auto_range_fit) {
1201                 /* If automatic range fitting is enabled. */
1202                 for (y = 0; y < ibuf->y; y++) {
1203                         const int Yu = y == (ibuf->y - 1) ? (ibuf->y - 1) : (y+1);
1204                         const int Yc = y;
1205                         const int Yd = y == 0 ? 0 : (y - 1);
1206
1207                         for (x= 0; x < ibuf->x; x++) {
1208                                 const int Xl = x == 0 ? 0 : (x - 1);
1209                                 const int Xc = x;
1210                                 const int Xr = x == (ibuf->x - 1) ? (ibuf->x - 1) : (x + 1);
1211
1212                                 const float Hcy = heights_buffer[Yc * ibuf->x + Xr] - heights_buffer[Yc * ibuf->x + Xl];
1213                                 const float Hu  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yu * ibuf->x + Xl];
1214                                 const float Hd  = heights_buffer[Yd * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xl];
1215
1216                                 const float Hl  = heights_buffer[Yu * ibuf->x + Xl] - heights_buffer[Yd * ibuf->x + Xl];
1217                                 const float Hcx = heights_buffer[Yu * ibuf->x + Xc] - heights_buffer[Yd * ibuf->x + Xc];
1218                                 const float Hr  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xr];
1219
1220                                 /* This corresponds to using the sobel kernel on the heights buffer
1221                                  * to obtain the derivative multiplied by 8.
1222                                  */
1223                                 const float deriv_x = Hu + 2 * Hcy + Hd;
1224                                 const float deriv_y = Hr + 2 * Hcx + Hl;
1225
1226                                 /* early out */
1227                                 index = ibuf->x * y + x;
1228                                 if (mask[index] != FILTER_MASK_USED) {
1229                                         continue;
1230                                 }
1231
1232                                 /* Widen bound. */
1233                                 if (fabsf(deriv_x) > max_num_deriv) {
1234                                         max_num_deriv = fabsf(deriv_x);
1235                                 }
1236
1237                                 if (fabsf(deriv_y) > max_num_deriv) {
1238                                         max_num_deriv = fabsf(deriv_y);
1239                                 }
1240                         }
1241                 }
1242         }
1243
1244         /* Output derivatives. */
1245         auto_range_fit &= (max_num_deriv > 0);
1246         for (y = 0; y < ibuf->y; y++) {
1247                 const int Yu= y==(ibuf->y-1) ? (ibuf->y-1) : (y+1);
1248                 const int Yc= y;
1249                 const int Yd= y==0 ? 0 : (y-1);
1250
1251                 for(x= 0; x<ibuf->x; x++) {
1252                         const int Xl = x == 0 ? 0 : (x - 1);
1253                         const int Xc = x;
1254                         const int Xr = x == (ibuf->x - 1) ? (ibuf->x - 1) : (x + 1);
1255
1256                         const float Hcy = heights_buffer[Yc * ibuf->x + Xr] - heights_buffer[Yc * ibuf->x + Xl];
1257                         const float Hu  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yu * ibuf->x + Xl];
1258                         const float Hd  = heights_buffer[Yd * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xl];
1259
1260                         const float Hl  = heights_buffer[Yu * ibuf->x + Xl] - heights_buffer[Yd * ibuf->x + Xl];
1261                         const float Hcx = heights_buffer[Yu * ibuf->x + Xc] - heights_buffer[Yd * ibuf->x + Xc];
1262                         const float Hr  = heights_buffer[Yu * ibuf->x + Xr] - heights_buffer[Yd * ibuf->x + Xr];
1263
1264                         /* This corresponds to using the sobel kernel on the heights buffer
1265                          * to obtain the derivative multiplied by 8.
1266                          */
1267                         float deriv_x = Hu + 2 * Hcy + Hd;
1268                         float deriv_y = Hr + 2 * Hcx + Hl;
1269
1270                         /* Early out. */
1271                         index = ibuf->x * y + x;
1272                         if (mask[index] != FILTER_MASK_USED){
1273                                 continue;
1274                         }
1275
1276                         if (auto_range_fit) {
1277                                 deriv_x /= max_num_deriv;
1278                                 deriv_y /= max_num_deriv;
1279                         } else {
1280                                 deriv_x *= (fmult / denom);
1281                                 deriv_y *= (fmult / denom);
1282                         }
1283
1284                         deriv_x = deriv_x * 0.5f + 0.5f;
1285                         deriv_y = deriv_y * 0.5f + 0.5f;
1286
1287                         /* Clamp. */
1288                         CLAMP(deriv_x, 0.0f, 1.0f);
1289                         CLAMP(deriv_y, 0.0f, 1.0f);
1290
1291                         /* Write out derivatives. */
1292                         if (ibuf->rect_float) {
1293                                 float *rrgbf = ibuf->rect_float + index * 4;
1294
1295                                 rrgbf[0] = deriv_x;
1296                                 rrgbf[1] = deriv_y;
1297                                 rrgbf[2] = 0.0f;
1298                                 rrgbf[3] = 1.0f;
1299                         } else {
1300                                 char *rrgb = (char*)ibuf->rect + index * 4;
1301
1302                                 rrgb[0] = FTOCHAR(deriv_x);
1303                                 rrgb[1] = FTOCHAR(deriv_y);
1304                                 rrgb[2] = 0;
1305                                 rrgb[3] = 255;
1306                         }
1307                 }
1308         }
1309
1310         /* Eeturn user-scale (for rendering). */
1311         return auto_range_fit ? (max_num_deriv / denom) : (fmult > 0.0f ? (1.0f / fmult) : 0.0f);
1312 }