fix [#34125] Crash when bake margin = 0
[blender.git] / source / blender / render / intern / source / bake.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
19  * All rights reserved.
20  *
21  * Contributors: 2004/2005/2006 Blender Foundation, full recode
22  * Contributors: Vertex color baking, Copyright 2011 AutoCRC
23  *
24  * ***** END GPL LICENSE BLOCK *****
25  */
26
27 /** \file blender/render/intern/source/bake.c
28  *  \ingroup render
29  */
30
31
32 /* system includes */
33 #include <stdio.h>
34 #include <string.h>
35
36 /* External modules: */
37 #include "MEM_guardedalloc.h"
38
39 #include "BLI_math.h"
40 #include "BLI_blenlib.h"
41 #include "BLI_threads.h"
42 #include "BLI_utildefines.h"
43
44 #include "DNA_image_types.h"
45 #include "DNA_material_types.h"
46 #include "DNA_mesh_types.h"
47 #include "DNA_meshdata_types.h"
48
49 #include "BKE_customdata.h"
50 #include "BKE_depsgraph.h"
51 #include "BKE_global.h"
52 #include "BKE_image.h"
53 #include "BKE_main.h"
54 #include "BKE_node.h"
55 #include "BKE_scene.h"
56
57 #include "IMB_imbuf_types.h"
58 #include "IMB_imbuf.h"
59 #include "IMB_colormanagement.h"
60
61 /* local include */
62 #include "rayintersection.h"
63 #include "rayobject.h"
64 #include "render_types.h"
65 #include "renderdatabase.h"
66 #include "shading.h"
67 #include "zbuf.h"
68
69 #include "PIL_time.h"
70
71 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
72 /* defined in pipeline.c, is hardcopy of active dynamic allocated Render */
73 /* only to be used here in this file, it's for speed */
74 extern struct Render R;
75 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
76
77
78 /* ************************* bake ************************ */
79
80
81 typedef struct BakeShade {
82         ShadeSample ssamp;
83         ObjectInstanceRen *obi;
84         VlakRen *vlr;
85         
86         ZSpan *zspan;
87         Image *ima;
88         ImBuf *ibuf;
89         
90         int rectx, recty, quad, type, vdone;
91         bool ready;
92
93         float dir[3];
94         Object *actob;
95
96         /* Output: vertex color or image data. If vcol is not NULL, rect and
97          * rect_float should be NULL. */
98         MPoly *mpoly;
99         MLoop *mloop;
100         MLoopCol *vcol;
101         
102         unsigned int *rect;
103         float *rect_float;
104
105         /* displacement buffer used for normalization with unknown maximal distance */
106         bool use_displacement_buffer;
107         float *displacement_buffer;
108         float displacement_min, displacement_max;
109         
110         bool use_mask;
111         char *rect_mask; /* bake pixel mask */
112
113         float dxco[3], dyco[3];
114
115         short *do_update;
116
117         struct ColorSpace *rect_colorspace;
118 } BakeShade;
119
120 static void bake_set_shade_input(ObjectInstanceRen *obi, VlakRen *vlr, ShadeInput *shi, int quad, int UNUSED(isect), int x, int y, float u, float v)
121 {
122         if (quad)
123                 shade_input_set_triangle_i(shi, obi, vlr, 0, 2, 3);
124         else
125                 shade_input_set_triangle_i(shi, obi, vlr, 0, 1, 2);
126                 
127         /* cache for shadow */
128         shi->samplenr = R.shadowsamplenr[shi->thread]++;
129
130         shi->mask = 0xFFFF; /* all samples */
131
132         shi->u = -u;
133         shi->v = -v;
134         shi->xs = x;
135         shi->ys = y;
136
137         shade_input_set_uv(shi);
138         shade_input_set_normals(shi);
139
140         /* no normal flip */
141         if (shi->flippednor)
142                 shade_input_flip_normals(shi);
143
144         /* set up view vector to look right at the surface (note that the normal
145          * is negated in the renderer so it does not need to be done here) */
146         shi->view[0] = shi->vn[0];
147         shi->view[1] = shi->vn[1];
148         shi->view[2] = shi->vn[2];
149 }
150
151 static void bake_shade(void *handle, Object *ob, ShadeInput *shi, int UNUSED(quad), int x, int y, float UNUSED(u), float UNUSED(v), float *tvn, float *ttang)
152 {
153         BakeShade *bs = handle;
154         ShadeSample *ssamp = &bs->ssamp;
155         ShadeResult shr;
156         VlakRen *vlr = shi->vlr;
157
158         shade_input_init_material(shi);
159
160         if (bs->type == RE_BAKE_AO) {
161                 ambient_occlusion(shi);
162
163                 if (R.r.bake_flag & R_BAKE_NORMALIZE) {
164                         copy_v3_v3(shr.combined, shi->ao);
165                 }
166                 else {
167                         zero_v3(shr.combined);
168                         environment_lighting_apply(shi, &shr);
169                 }
170         }
171         else {
172                 if (bs->type == RE_BAKE_SHADOW) /* Why do shadows set the color anyhow?, ignore material color for baking */
173                         shi->r = shi->g = shi->b = 1.0f;
174         
175                 shade_input_set_shade_texco(shi);
176                 
177                 /* only do AO for a full bake (and obviously AO bakes)
178                  * AO for light bakes is a leftover and might not be needed */
179                 if (ELEM3(bs->type, RE_BAKE_ALL, RE_BAKE_AO, RE_BAKE_LIGHT))
180                         shade_samples_do_AO(ssamp);
181                 
182                 if (shi->mat->nodetree && shi->mat->use_nodes) {
183                         ntreeShaderExecTree(shi->mat->nodetree, shi, &shr);
184                         shi->mat = vlr->mat;  /* shi->mat is being set in nodetree */
185                 }
186                 else
187                         shade_material_loop(shi, &shr);
188
189                 if (bs->type == RE_BAKE_NORMALS) {
190                         float nor[3];
191
192                         copy_v3_v3(nor, shi->vn);
193
194                         if (R.r.bake_normal_space == R_BAKE_SPACE_CAMERA) {
195                                 /* pass */
196                         }
197                         else if (R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
198                                 float mat[3][3], imat[3][3];
199
200                                 /* bitangent */
201                                 if (tvn && ttang) {
202                                         copy_v3_v3(mat[0], ttang);
203                                         cross_v3_v3v3(mat[1], tvn, ttang);
204                                         mul_v3_fl(mat[1], ttang[3]);
205                                         copy_v3_v3(mat[2], tvn);
206                                 }
207                                 else {
208                                         copy_v3_v3(mat[0], shi->nmaptang);
209                                         cross_v3_v3v3(mat[1], shi->nmapnorm, shi->nmaptang);
210                                         mul_v3_fl(mat[1], shi->nmaptang[3]);
211                                         copy_v3_v3(mat[2], shi->nmapnorm);
212                                 }
213
214                                 invert_m3_m3(imat, mat);
215                                 mul_m3_v3(imat, nor);
216                         }
217                         else if (R.r.bake_normal_space == R_BAKE_SPACE_OBJECT)
218                                 mul_mat3_m4_v3(ob->imat_ren, nor);  /* ob->imat_ren includes viewinv! */
219                         else if (R.r.bake_normal_space == R_BAKE_SPACE_WORLD)
220                                 mul_mat3_m4_v3(R.viewinv, nor);
221
222                         normalize_v3(nor); /* in case object has scaling */
223
224                         /* The invert of the red channel is to make
225                          * the normal map compliant with the outside world.
226                          * It needs to be done because in Blender
227                          * the normal used in the renderer points inward. It is generated
228                          * this way in calc_vertexnormals(). Should this ever change
229                          * this negate must be removed. */
230                         shr.combined[0] = (-nor[0]) / 2.0f + 0.5f;
231                         shr.combined[1] = nor[1]    / 2.0f + 0.5f;
232                         shr.combined[2] = nor[2]    / 2.0f + 0.5f;
233                 }
234                 else if (bs->type == RE_BAKE_TEXTURE) {
235                         copy_v3_v3(shr.combined, &shi->r);
236                         shr.alpha = shi->alpha;
237                 }
238                 else if (bs->type == RE_BAKE_SHADOW) {
239                         copy_v3_v3(shr.combined, shr.shad);
240                         shr.alpha = shi->alpha;
241                 }
242                 else if (bs->type == RE_BAKE_SPEC_COLOR) {
243                         copy_v3_v3(shr.combined, &shi->specr);
244                         shr.alpha = 1.0f;
245                 }
246                 else if (bs->type == RE_BAKE_SPEC_INTENSITY) {
247                         copy_v3_fl(shr.combined, shi->spec);
248                         shr.alpha = 1.0f;
249                 }
250                 else if (bs->type == RE_BAKE_MIRROR_COLOR) {
251                         copy_v3_v3(shr.combined, &shi->mirr);
252                         shr.alpha = 1.0f;
253                 }
254                 else if (bs->type == RE_BAKE_MIRROR_INTENSITY) {
255                         copy_v3_fl(shr.combined, shi->ray_mirror);
256                         shr.alpha = 1.0f;
257                 }
258                 else if (bs->type == RE_BAKE_ALPHA) {
259                         copy_v3_fl(shr.combined, shi->alpha);
260                         shr.alpha = 1.0f;
261                 }
262                 else if (bs->type == RE_BAKE_EMIT) {
263                         copy_v3_fl(shr.combined, shi->emit);
264                         shr.alpha = 1.0f;
265                 }
266         }
267         
268         if (bs->rect_float && !bs->vcol) {
269                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
270                 copy_v3_v3(col, shr.combined);
271                 if (bs->type == RE_BAKE_ALL || bs->type == RE_BAKE_TEXTURE) {
272                         col[3] = shr.alpha;
273                 }
274                 else {
275                         col[3] = 1.0;
276                 }
277         }
278         else {
279                 /* Target is char (LDR). */
280                 unsigned char col[4];
281
282                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
283                         float rgb[3];
284
285                         copy_v3_v3(rgb, shr.combined);
286                         if (R.scene_color_manage) {
287                                 /* Vertex colors have no way to specify color space, so they
288                                  * default to sRGB. */
289                                 if (!bs->vcol)
290                                         IMB_colormanagement_scene_linear_to_colorspace_v3(rgb, bs->rect_colorspace);
291                                 else
292                                         linearrgb_to_srgb_v3_v3(rgb, rgb);
293                         }
294                         rgb_float_to_uchar(col, rgb);
295                 }
296                 else {
297                         rgb_float_to_uchar(col, shr.combined);
298                 }
299                 
300                 if (ELEM(bs->type, RE_BAKE_ALL, RE_BAKE_TEXTURE)) {
301                         col[3] = FTOCHAR(shr.alpha);
302                 }
303                 else {
304                         col[3] = 255;
305                 }
306
307                 if (bs->vcol) {
308                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
309                          * but is used only for vertex painting). */
310                         bs->vcol->r = col[0];
311                         bs->vcol->g = col[1];
312                         bs->vcol->b = col[2];
313                 }
314                 else {
315                         unsigned char *imcol = (unsigned char *)(bs->rect + bs->rectx * y + x);
316                         copy_v4_v4_char((char *)imcol, (char *)col);
317                 }
318
319         }
320         
321         if (bs->rect_mask) {
322                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
323         }
324 }
325
326 static void bake_displacement(void *handle, ShadeInput *UNUSED(shi), float dist, int x, int y)
327 {
328         BakeShade *bs = handle;
329         float disp;
330
331         if (R.r.bake_flag & R_BAKE_NORMALIZE) {
332                 if (R.r.bake_maxdist)
333                         disp = (dist + R.r.bake_maxdist) / (R.r.bake_maxdist * 2);  /* alter the range from [-bake_maxdist, bake_maxdist] to [0, 1]*/
334                 else
335                         disp = dist;
336         }
337         else {
338                 disp = 0.5f + dist; /* alter the range from [-0.5,0.5] to [0,1]*/
339         }
340
341         if (bs->displacement_buffer) {
342                 float *displacement = bs->displacement_buffer + (bs->rectx * y + x);
343                 *displacement = disp;
344                 bs->displacement_min = min_ff(bs->displacement_min, disp);
345                 bs->displacement_max = max_ff(bs->displacement_max, disp);
346         }
347
348         if (bs->rect_float && !bs->vcol) {
349                 float *col = bs->rect_float + 4 * (bs->rectx * y + x);
350                 col[0] = col[1] = col[2] = disp;
351                 col[3] = 1.0f;
352         }
353         else {
354                 /* Target is char (LDR). */
355                 unsigned char col[4];
356                 col[0] = col[1] = col[2] = FTOCHAR(disp);
357                 col[3] = 255;
358
359                 if (bs->vcol) {
360                         /* Vertex color baking. Vcol has no useful alpha channel (it exists
361                          * but is used only for vertex painting). */
362                         bs->vcol->r = col[0];
363                         bs->vcol->g = col[1];
364                         bs->vcol->b = col[2];
365                 }
366                 else {
367                         char *imcol = (char *)(bs->rect + bs->rectx * y + x);
368                         copy_v4_v4_char((char *)imcol, (char *)col);
369                 }
370         }
371         if (bs->rect_mask) {
372                 bs->rect_mask[bs->rectx * y + x] = FILTER_MASK_USED;
373         }
374 }
375
376 static int bake_intersect_tree(RayObject *raytree, Isect *isect, float *start, float *dir, float sign, float *hitco, float *dist)
377 {
378         float maxdist;
379         int hit;
380
381         /* might be useful to make a user setting for maxsize*/
382         if (R.r.bake_maxdist > 0.0f)
383                 maxdist = R.r.bake_maxdist;
384         else
385                 maxdist = RE_RAYTRACE_MAXDIST + R.r.bake_biasdist;
386
387         /* 'dir' is always normalized */
388         madd_v3_v3v3fl(isect->start, start, dir, -R.r.bake_biasdist);
389
390         mul_v3_v3fl(isect->dir, dir, sign);
391
392         isect->dist = maxdist;
393
394         hit = RE_rayobject_raycast(raytree, isect);
395         if (hit) {
396                 madd_v3_v3v3fl(hitco, isect->start, isect->dir, isect->dist);
397
398                 *dist = isect->dist;
399         }
400
401         return hit;
402 }
403
404 static void bake_set_vlr_dxyco(BakeShade *bs, float *uv1, float *uv2, float *uv3)
405 {
406         VlakRen *vlr = bs->vlr;
407         float A, d1, d2, d3, *v1, *v2, *v3;
408
409         if (bs->quad) {
410                 v1 = vlr->v1->co;
411                 v2 = vlr->v3->co;
412                 v3 = vlr->v4->co;
413         }
414         else {
415                 v1 = vlr->v1->co;
416                 v2 = vlr->v2->co;
417                 v3 = vlr->v3->co;
418         }
419
420         /* formula derived from barycentric coordinates:
421          * (uvArea1*v1 + uvArea2*v2 + uvArea3*v3)/uvArea
422          * then taking u and v partial derivatives to get dxco and dyco */
423         A = (uv2[0] - uv1[0]) * (uv3[1] - uv1[1]) - (uv3[0] - uv1[0]) * (uv2[1] - uv1[1]);
424
425         if (fabsf(A) > FLT_EPSILON) {
426                 A = 0.5f / A;
427
428                 d1 = uv2[1] - uv3[1];
429                 d2 = uv3[1] - uv1[1];
430                 d3 = uv1[1] - uv2[1];
431                 bs->dxco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
432                 bs->dxco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
433                 bs->dxco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
434
435                 d1 = uv3[0] - uv2[0];
436                 d2 = uv1[0] - uv3[0];
437                 d3 = uv2[0] - uv1[0];
438                 bs->dyco[0] = (v1[0] * d1 + v2[0] * d2 + v3[0] * d3) * A;
439                 bs->dyco[1] = (v1[1] * d1 + v2[1] * d2 + v3[1] * d3) * A;
440                 bs->dyco[2] = (v1[2] * d1 + v2[2] * d2 + v3[2] * d3) * A;
441         }
442         else {
443                 bs->dxco[0] = bs->dxco[1] = bs->dxco[2] = 0.0f;
444                 bs->dyco[0] = bs->dyco[1] = bs->dyco[2] = 0.0f;
445         }
446
447         if (bs->obi->flag & R_TRANSFORMED) {
448                 mul_m3_v3(bs->obi->nmat, bs->dxco);
449                 mul_m3_v3(bs->obi->nmat, bs->dyco);
450         }
451 }
452
453 static void do_bake_shade(void *handle, int x, int y, float u, float v)
454 {
455         BakeShade *bs = handle;
456         VlakRen *vlr = bs->vlr;
457         ObjectInstanceRen *obi = bs->obi;
458         Object *ob = obi->obr->ob;
459         float l, *v1, *v2, *v3, tvn[3], ttang[4];
460         int quad;
461         ShadeSample *ssamp = &bs->ssamp;
462         ShadeInput *shi = ssamp->shi;
463
464         /* fast threadsafe break test */
465         if (R.test_break(R.tbh))
466                 return;
467
468         /* setup render coordinates */
469         if (bs->quad) {
470                 v1 = vlr->v1->co;
471                 v2 = vlr->v3->co;
472                 v3 = vlr->v4->co;
473         }
474         else {
475                 v1 = vlr->v1->co;
476                 v2 = vlr->v2->co;
477                 v3 = vlr->v3->co;
478         }
479
480         l = 1.0f - u - v;
481
482         /* shrink barycentric coordinates inwards slightly to avoid some issues
483          * where baking selected to active might just miss the other face at the
484          * near the edge of a face */
485         if (bs->actob) {
486                 const float eps = 1.0f - 1e-4f;
487                 float invsum;
488
489                 u = (u - 0.5f) * eps + 0.5f;
490                 v = (v - 0.5f) * eps + 0.5f;
491                 l = (l - 0.5f) * eps + 0.5f;
492
493                 invsum = 1.0f / (u + v + l);
494
495                 u *= invsum;
496                 v *= invsum;
497                 l *= invsum;
498         }
499
500         /* renderco */
501         shi->co[0] = l * v3[0] + u * v1[0] + v * v2[0];
502         shi->co[1] = l * v3[1] + u * v1[1] + v * v2[1];
503         shi->co[2] = l * v3[2] + u * v1[2] + v * v2[2];
504
505         /* avoid self shadow with vertex bake from adjacent faces [#33729] */
506         if ((bs->vcol != NULL) && (bs->actob == NULL)) {
507                 madd_v3_v3fl(shi->co, vlr->n, 0.0001f);
508         }
509
510         if (obi->flag & R_TRANSFORMED)
511                 mul_m4_v3(obi->mat, shi->co);
512
513         copy_v3_v3(shi->dxco, bs->dxco);
514         copy_v3_v3(shi->dyco, bs->dyco);
515
516         quad = bs->quad;
517         bake_set_shade_input(obi, vlr, shi, quad, 0, x, y, u, v);
518
519         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT) {
520                 shade_input_set_shade_texco(shi);
521                 copy_v3_v3(tvn, shi->nmapnorm);
522                 copy_v4_v4(ttang, shi->nmaptang);
523         }
524
525         /* if we are doing selected to active baking, find point on other face */
526         if (bs->actob) {
527                 Isect isec, minisec;
528                 float co[3], minco[3], dist, mindist = 0.0f;
529                 int hit, sign, dir = 1;
530
531                 /* intersect with ray going forward and backward*/
532                 hit = 0;
533                 memset(&minisec, 0, sizeof(minisec));
534                 minco[0] = minco[1] = minco[2] = 0.0f;
535
536                 copy_v3_v3(bs->dir, shi->vn);
537
538                 for (sign = -1; sign <= 1; sign += 2) {
539                         memset(&isec, 0, sizeof(isec));
540                         isec.mode = RE_RAY_MIRROR;
541
542                         isec.orig.ob   = obi;
543                         isec.orig.face = vlr;
544                         isec.userdata = bs->actob;
545                         isec.check = RE_CHECK_VLR_BAKE;
546                         isec.skip = RE_SKIP_VLR_NEIGHBOUR;
547
548                         if (bake_intersect_tree(R.raytree, &isec, shi->co, shi->vn, sign, co, &dist)) {
549                                 if (!hit || len_squared_v3v3(shi->co, co) < len_squared_v3v3(shi->co, minco)) {
550                                         minisec = isec;
551                                         mindist = dist;
552                                         copy_v3_v3(minco, co);
553                                         hit = 1;
554                                         dir = sign;
555                                 }
556                         }
557                 }
558
559                 if (bs->type == RE_BAKE_DISPLACEMENT) {
560                         if (hit)
561                                 bake_displacement(handle, shi, (dir == -1) ? mindist : -mindist, x, y);
562                         else
563                                 bake_displacement(handle, shi, 0.0f, x, y);
564                         return;
565                 }
566
567                 /* if hit, we shade from the new point, otherwise from point one starting face */
568                 if (hit) {
569                         obi = (ObjectInstanceRen *)minisec.hit.ob;
570                         vlr = (VlakRen *)minisec.hit.face;
571                         quad = (minisec.isect == 2);
572                         copy_v3_v3(shi->co, minco);
573
574                         u = -minisec.u;
575                         v = -minisec.v;
576                         bake_set_shade_input(obi, vlr, shi, quad, 1, x, y, u, v);
577                 }
578         }
579
580         if (bs->type == RE_BAKE_NORMALS && R.r.bake_normal_space == R_BAKE_SPACE_TANGENT)
581                 bake_shade(handle, ob, shi, quad, x, y, u, v, tvn, ttang);
582         else
583                 bake_shade(handle, ob, shi, quad, x, y, u, v, 0, 0);
584 }
585
586 static int get_next_bake_face(BakeShade *bs)
587 {
588         ObjectRen *obr;
589         VlakRen *vlr;
590         MTFace *tface;
591         static int v = 0, vdone = false;
592         static ObjectInstanceRen *obi = NULL;
593
594         if (bs == NULL) {
595                 vlr = NULL;
596                 v = vdone = false;
597                 obi = R.instancetable.first;
598                 return 0;
599         }
600         
601         BLI_lock_thread(LOCK_CUSTOM1);
602
603         for (; obi; obi = obi->next, v = 0) {
604                 obr = obi->obr;
605
606                 for (; v < obr->totvlak; v++) {
607                         vlr = RE_findOrAddVlak(obr, v);
608
609                         if ((bs->actob && bs->actob == obr->ob) || (!bs->actob && (obr->ob->flag & SELECT))) {
610                                 if (R.r.bake_flag & R_BAKE_VCOL) {
611                                         /* Gather face data for vertex color bake */
612                                         Mesh *me;
613                                         int *origindex, vcollayer;
614                                         CustomDataLayer *cdl;
615
616                                         if (obr->ob->type != OB_MESH)
617                                                 continue;
618                                         me = obr->ob->data;
619
620                                         origindex = RE_vlakren_get_origindex(obr, vlr, 0);
621                                         if (origindex == NULL)
622                                                 continue;
623                                         if (*origindex >= me->totpoly) {
624                                                 /* Small hack for Array modifier, which gives false
625                                                    original indices - z0r */
626                                                 continue;
627                                         }
628 #if 0
629                                         /* Only shade selected faces. */
630                                         if ((me->mface[*origindex].flag & ME_FACE_SEL) == 0)
631                                                 continue;
632 #endif
633
634                                         vcollayer = CustomData_get_render_layer_index(&me->ldata, CD_MLOOPCOL);
635                                         if (vcollayer == -1)
636                                                 continue;
637
638                                         cdl = &me->ldata.layers[vcollayer];
639                                         bs->mpoly = me->mpoly + *origindex;
640                                         bs->vcol = ((MLoopCol *)cdl->data) + bs->mpoly->loopstart;
641                                         bs->mloop = me->mloop + bs->mpoly->loopstart;
642
643                                         /* Tag mesh for reevaluation. */
644                                         DAG_id_tag_update(&me->id, 0);
645                                 }
646                                 else {
647                                         Image *ima = NULL;
648                                         ImBuf *ibuf = NULL;
649                                         const float vec_alpha[4] = {0.0f, 0.0f, 0.0f, 0.0f};
650                                         const float vec_solid[4] = {0.0f, 0.0f, 0.0f, 1.0f};
651
652                                         tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
653
654                                         if (!tface || !tface->tpage)
655                                                 continue;
656
657                                         ima = tface->tpage;
658                                         ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
659
660                                         if (ibuf == NULL)
661                                                 continue;
662
663                                         if (ibuf->rect == NULL && ibuf->rect_float == NULL) {
664                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
665                                                 continue;
666                                         }
667
668                                         if (ibuf->rect_float && !(ibuf->channels == 0 || ibuf->channels == 4)) {
669                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
670                                                 continue;
671                                         }
672                                         
673                                         if (ima->flag & IMA_USED_FOR_RENDER) {
674                                                 ima->id.flag &= ~LIB_DOIT;
675                                                 BKE_image_release_ibuf(ima, ibuf, NULL);
676                                                 continue;
677                                         }
678                                         
679                                         /* find the image for the first time? */
680                                         if (ima->id.flag & LIB_DOIT) {
681                                                 ima->id.flag &= ~LIB_DOIT;
682                                                 
683                                                 /* we either fill in float or char, this ensures things go fine */
684                                                 if (ibuf->rect_float)
685                                                         imb_freerectImBuf(ibuf);
686                                                 /* clear image */
687                                                 if (R.r.bake_flag & R_BAKE_CLEAR)
688                                                         IMB_rectfill(ibuf, (ibuf->planes == R_IMF_PLANES_RGBA) ? vec_alpha : vec_solid);
689
690                                                 /* might be read by UI to set active image for display */
691                                                 R.bakebuf = ima;
692                                         }
693
694                                         /* Tag image for redraw. */
695                                         ibuf->userflags |= IB_DISPLAY_BUFFER_INVALID;
696                                         BKE_image_release_ibuf(ima, ibuf, NULL);
697                                 }
698
699                                 bs->obi = obi;
700                                 bs->vlr = vlr;
701                                 bs->vdone++;  /* only for error message if nothing was rendered */
702                                 v++;
703                                 BLI_unlock_thread(LOCK_CUSTOM1);
704                                 return 1;
705                         }
706                 }
707         }
708         
709         BLI_unlock_thread(LOCK_CUSTOM1);
710         return 0;
711 }
712
713 static void bake_single_vertex(BakeShade *bs, VertRen *vert, float u, float v)
714 {
715         int *origindex, i;
716         MLoopCol *basevcol;
717         MLoop *mloop;
718
719         origindex = RE_vertren_get_origindex(bs->obi->obr, vert, 0);
720         if (!origindex || *origindex == ORIGINDEX_NONE)
721                 return;
722
723         /* Search for matching vertex index and apply shading. */
724         for (i = 0; i < bs->mpoly->totloop; i++) {
725                 mloop = bs->mloop + i;
726                 if (mloop->v != *origindex)
727                         continue;
728                 basevcol = bs->vcol;
729                 bs->vcol = basevcol + i;
730                 do_bake_shade(bs, 0, 0, u, v);
731                 bs->vcol = basevcol;
732                 break;
733         }
734 }
735
736 /* Bake all vertices of a face. Actually, this still works on a face-by-face
737  * basis, and each vertex on each face is shaded. Vertex colors are a property
738  * of loops, not vertices. */
739 static void shade_verts(BakeShade *bs)
740 {
741         VlakRen *vlr = bs->vlr;
742
743         /* Disable baking to image; write to vcol instead. vcol pointer is set in
744          * bake_single_vertex. */
745         bs->ima = NULL;
746         bs->rect = NULL;
747         bs->rect_float = NULL;
748         bs->displacement_buffer = NULL;
749         bs->displacement_min = FLT_MAX;
750         bs->displacement_max = -FLT_MAX;
751
752         bs->quad = 0;
753
754         /* No anti-aliasing for vertices. */
755         zero_v3(bs->dxco);
756         zero_v3(bs->dyco);
757
758         /* Shade each vertex of the face. u and v are barycentric coordinates; since
759          * we're only interested in vertices, these will be 0 or 1. */
760         if ((vlr->flag & R_FACE_SPLIT) == 0) {
761                 /* Processing triangle face, whole quad, or first half of split quad. */
762
763                 bake_single_vertex(bs, bs->vlr->v1, 1.0f, 0.0f);
764                 bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
765                 bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
766
767                 if (vlr->v4) {
768                         bs->quad = 1;
769                         bake_single_vertex(bs, bs->vlr->v4, 0.0f, 0.0f);
770                 }
771         }
772         else {
773                 /* Processing second half of split quad. Only one vertex to go. */
774                 if (vlr->flag & R_DIVIDE_24) {
775                         bake_single_vertex(bs, bs->vlr->v2, 0.0f, 1.0f);
776                 }
777                 else {
778                         bake_single_vertex(bs, bs->vlr->v3, 0.0f, 0.0f);
779                 }
780         }
781 }
782
783 /* already have tested for tface and ima and zspan */
784 static void shade_tface(BakeShade *bs)
785 {
786         VlakRen *vlr = bs->vlr;
787         ObjectInstanceRen *obi = bs->obi;
788         ObjectRen *obr = obi->obr;
789         MTFace *tface = RE_vlakren_get_tface(obr, vlr, obr->bakemtface, NULL, 0);
790         Image *ima = tface->tpage;
791         float vec[4][2];
792         int a, i1, i2, i3;
793         
794         /* check valid zspan */
795         if (ima != bs->ima) {
796                 BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
797
798                 bs->ima = ima;
799                 bs->ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
800                 /* note, these calls only free/fill contents of zspan struct, not zspan itself */
801                 zbuf_free_span(bs->zspan);
802                 zbuf_alloc_span(bs->zspan, bs->ibuf->x, bs->ibuf->y, R.clipcrop);
803         }
804
805         bs->rectx = bs->ibuf->x;
806         bs->recty = bs->ibuf->y;
807         bs->rect = bs->ibuf->rect;
808         bs->rect_colorspace = bs->ibuf->rect_colorspace;
809         bs->rect_float = bs->ibuf->rect_float;
810         bs->vcol = NULL;
811         bs->quad = 0;
812         bs->rect_mask = NULL;
813         bs->displacement_buffer = NULL;
814
815         if (bs->use_mask || bs->use_displacement_buffer) {
816                 BakeImBufuserData *userdata = bs->ibuf->userdata;
817                 if (userdata == NULL) {
818                         BLI_lock_thread(LOCK_CUSTOM1);
819                         userdata = bs->ibuf->userdata;
820                         if (userdata == NULL) /* since the thread was locked, its possible another thread alloced the value */
821                                 userdata = MEM_callocN(sizeof(BakeImBufuserData), "BakeImBufuserData");
822
823                         if (bs->use_mask) {
824                                 if (userdata->mask_buffer == NULL) {
825                                         userdata->mask_buffer = MEM_callocN(sizeof(char) * bs->rectx * bs->recty, "BakeMask");
826                                 }
827                         }
828
829                         if (bs->use_displacement_buffer)
830                                 userdata->displacement_buffer = MEM_callocN(sizeof(float) * bs->rectx * bs->recty, "BakeDisp");
831
832                         bs->ibuf->userdata = userdata;
833
834                         BLI_unlock_thread(LOCK_CUSTOM1);
835                 }
836
837                 bs->rect_mask = userdata->mask_buffer;
838                 bs->displacement_buffer = userdata->displacement_buffer;
839         }
840         
841         /* get pixel level vertex coordinates */
842         for (a = 0; a < 4; a++) {
843                 /* Note, workaround for pixel aligned UVs which are common and can screw up our intersection tests
844                  * where a pixel gets in between 2 faces or the middle of a quad,
845                  * camera aligned quads also have this problem but they are less common.
846                  * Add a small offset to the UVs, fixes bug #18685 - Campbell */
847                 vec[a][0] = tface->uv[a][0] * (float)bs->rectx - (0.5f + 0.001f);
848                 vec[a][1] = tface->uv[a][1] * (float)bs->recty - (0.5f + 0.002f);
849         }
850
851         /* UV indices have to be corrected for possible quad->tria splits */
852         i1 = 0; i2 = 1; i3 = 2;
853         vlr_set_uv_indices(vlr, &i1, &i2, &i3);
854         bake_set_vlr_dxyco(bs, vec[i1], vec[i2], vec[i3]);
855         zspan_scanconvert(bs->zspan, bs, vec[i1], vec[i2], vec[i3], do_bake_shade);
856         
857         if (vlr->v4) {
858                 bs->quad = 1;
859                 bake_set_vlr_dxyco(bs, vec[0], vec[2], vec[3]);
860                 zspan_scanconvert(bs->zspan, bs, vec[0], vec[2], vec[3], do_bake_shade);
861         }
862 }
863
864 static void *do_bake_thread(void *bs_v)
865 {
866         BakeShade *bs = bs_v;
867
868         while (get_next_bake_face(bs)) {
869                 if (R.r.bake_flag & R_BAKE_VCOL) {
870                         shade_verts(bs);
871                 }
872                 else {
873                         shade_tface(bs);
874                 }
875                 
876                 /* fast threadsafe break test */
877                 if (R.test_break(R.tbh))
878                         break;
879
880                 /* access is not threadsafe but since its just true/false probably ok
881                  * only used for interactive baking */
882                 if (bs->do_update) {
883                         *bs->do_update = true;
884                 }
885         }
886         bs->ready = true;
887
888         BKE_image_release_ibuf(bs->ima, bs->ibuf, NULL);
889
890         return NULL;
891 }
892
893 void RE_bake_ibuf_filter(ImBuf *ibuf, char *mask, const int filter)
894 {
895         /* must check before filtering */
896         const short is_new_alpha = (ibuf->planes != R_IMF_PLANES_RGBA) && BKE_imbuf_alpha_test(ibuf);
897
898         /* Margin */
899         if (filter) {
900                 IMB_filter_extend(ibuf, mask, filter);
901         }
902
903         /* if the bake results in new alpha then change the image setting */
904         if (is_new_alpha) {
905                 ibuf->planes = R_IMF_PLANES_RGBA;
906         }
907         else {
908                 if (filter && ibuf->planes != R_IMF_PLANES_RGBA) {
909                         /* clear alpha added by filtering */
910                         IMB_rectfill_alpha(ibuf, 1.0f);
911                 }
912         }
913 }
914
915 void RE_bake_ibuf_normalize_displacement(ImBuf *ibuf, float *displacement, char *mask, float displacement_min, float displacement_max)
916 {
917         int i;
918         float *current_displacement = displacement;
919         char *current_mask = mask;
920         float max_distance;
921
922         max_distance = max_ff(fabsf(displacement_min), fabsf(displacement_max));
923
924         for (i = 0; i < ibuf->x * ibuf->y; i++) {
925                 if (*current_mask == FILTER_MASK_USED) {
926                         float normalized_displacement;
927
928                         if (max_distance > 1e-5f)
929                                 normalized_displacement = (*current_displacement + max_distance) / (max_distance * 2);
930                         else
931                                 normalized_displacement = 0.5f;
932
933                         if (ibuf->rect_float) {
934                                 /* currently baking happens to RGBA only */
935                                 float *fp = ibuf->rect_float + i * 4;
936                                 fp[0] = fp[1] = fp[2] = normalized_displacement;
937                                 fp[3] = 1.0f;
938                         }
939
940                         if (ibuf->rect) {
941                                 unsigned char *cp = (unsigned char *) (ibuf->rect + i);
942                                 cp[0] = cp[1] = cp[2] = FTOCHAR(normalized_displacement);
943                                 cp[3] = 255;
944                         }
945                 }
946
947                 current_displacement++;
948                 current_mask++;
949         }
950 }
951
952 /* using object selection tags, the faces with UV maps get baked */
953 /* render should have been setup */
954 /* returns 0 if nothing was handled */
955 int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_update, float *progress)
956 {
957         BakeShade *handles;
958         ListBase threads;
959         Image *ima;
960         int a, vdone = false, result = BAKE_RESULT_OK;
961         bool use_mask = false;
962         bool use_displacement_buffer = false;
963         
964         re->scene_color_manage = BKE_scene_check_color_management_enabled(re->scene);
965         
966         /* initialize render global */
967         R = *re;
968         R.bakebuf = NULL;
969
970         /* initialize static vars */
971         get_next_bake_face(NULL);
972         
973         /* do we need a mask? */
974         if (re->r.bake_filter)
975                 use_mask = true;
976
977         /* do we need buffer to store displacements  */
978         if (type == RE_BAKE_DISPLACEMENT) {
979                 if ((R.r.bake_flag & R_BAKE_NORMALIZE) && R.r.bake_maxdist == 0.0f) {
980                         use_displacement_buffer = true;
981                         use_mask = true;
982                 }
983         }
984
985         /* baker uses this flag to detect if image was initialized */
986         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
987                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
988                         ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
989                         ima->id.flag |= LIB_DOIT;
990                         ima->flag &= ~IMA_USED_FOR_RENDER;
991                         if (ibuf) {
992                                 ibuf->userdata = NULL; /* use for masking if needed */
993                         }
994                         BKE_image_release_ibuf(ima, ibuf, NULL);
995                 }
996         }
997
998         BLI_init_threads(&threads, do_bake_thread, re->r.threads);
999
1000         handles = MEM_callocN(sizeof(BakeShade) * re->r.threads, "BakeShade");
1001
1002         /* get the threads running */
1003         for (a = 0; a < re->r.threads; a++) {
1004                 /* set defaults in handles */
1005                 handles[a].ssamp.shi[0].lay = re->lay;
1006
1007                 if (type == RE_BAKE_SHADOW) {
1008                         handles[a].ssamp.shi[0].passflag = SCE_PASS_SHADOW;
1009                 }
1010                 else {
1011                         handles[a].ssamp.shi[0].passflag = SCE_PASS_COMBINED;
1012                 }
1013                 handles[a].ssamp.shi[0].combinedflag = ~(SCE_PASS_SPEC);
1014                 handles[a].ssamp.shi[0].thread = a;
1015                 handles[a].ssamp.tot = 1;
1016
1017                 handles[a].type = type;
1018                 handles[a].actob = actob;
1019                 if (R.r.bake_flag & R_BAKE_VCOL)
1020                         handles[a].zspan = NULL;
1021                 else
1022                         handles[a].zspan = MEM_callocN(sizeof(ZSpan), "zspan for bake");
1023                 
1024                 handles[a].use_mask = use_mask;
1025                 handles[a].use_displacement_buffer = use_displacement_buffer;
1026
1027                 handles[a].do_update = do_update; /* use to tell the view to update */
1028                 
1029                 handles[a].displacement_min = FLT_MAX;
1030                 handles[a].displacement_max = -FLT_MAX;
1031
1032                 BLI_insert_thread(&threads, &handles[a]);
1033         }
1034         
1035         /* wait for everything to be done */
1036         a = 0;
1037         while (a != re->r.threads) {
1038                 PIL_sleep_ms(50);
1039
1040                 /* calculate progress */
1041                 for (vdone = false, a = 0; a < re->r.threads; a++)
1042                         vdone += handles[a].vdone;
1043                 if (progress)
1044                         *progress = (float)(vdone / (float)re->totvlak);
1045
1046                 for (a = 0; a < re->r.threads; a++) {
1047                         if (handles[a].ready == false) {
1048                                 break;
1049                         }
1050                 }
1051         }
1052
1053         /* filter and refresh images */
1054         if ((R.r.bake_flag & R_BAKE_VCOL) == 0) {
1055                 float displacement_min = FLT_MAX, displacement_max = -FLT_MAX;
1056
1057                 if (use_displacement_buffer) {
1058                         for (a = 0; a < re->r.threads; a++) {
1059                                 displacement_min = min_ff(displacement_min, handles[a].displacement_min);
1060                                 displacement_max = max_ff(displacement_max, handles[a].displacement_max);
1061                         }
1062                 }
1063
1064                 for (ima = G.main->image.first; ima; ima = ima->id.next) {
1065                         if ((ima->id.flag & LIB_DOIT) == 0) {
1066                                 ImBuf *ibuf = BKE_image_acquire_ibuf(ima, NULL, NULL);
1067                                 BakeImBufuserData *userdata;
1068
1069                                 if (ima->flag & IMA_USED_FOR_RENDER)
1070                                         result = BAKE_RESULT_FEEDBACK_LOOP;
1071
1072                                 if (!ibuf)
1073                                         continue;
1074
1075                                 if (userdata) {
1076                                         userdata = (BakeImBufuserData *) ibuf->userdata;
1077                                         RE_bake_ibuf_filter(ibuf, userdata->mask_buffer, re->r.bake_filter);
1078
1079                                         if (use_displacement_buffer) {
1080                                                 RE_bake_ibuf_normalize_displacement(ibuf, userdata->displacement_buffer, userdata->mask_buffer,
1081                                                                                     displacement_min, displacement_max);
1082                                         }
1083                                 }
1084
1085                                 ibuf->userflags |= IB_BITMAPDIRTY;
1086                                 BKE_image_release_ibuf(ima, ibuf, NULL);
1087                         }
1088                 }
1089
1090                 /* calculate return value */
1091                 for (a = 0; a < re->r.threads; a++) {
1092                         zbuf_free_span(handles[a].zspan);
1093                         MEM_freeN(handles[a].zspan);
1094                 }
1095         }
1096
1097         MEM_freeN(handles);
1098         
1099         BLI_end_threads(&threads);
1100
1101         if (vdone == 0) {
1102                 result = BAKE_RESULT_NO_OBJECTS;
1103         }
1104
1105         return result;
1106 }
1107
1108 struct Image *RE_bake_shade_get_image(void)
1109 {
1110         return R.bakebuf;
1111 }
1112