Cleanup
[blender.git] / source / blender / blenkernel / intern / paint.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software  Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2009 by Nicholas Bishop
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/paint.c
29  *  \ingroup bke
30  */
31
32 #include <stdlib.h>
33 #include <string.h>
34
35 #include "MEM_guardedalloc.h"
36
37 #include "DNA_object_types.h"
38 #include "DNA_mesh_types.h"
39 #include "DNA_meshdata_types.h"
40 #include "DNA_modifier_types.h"
41 #include "DNA_scene_types.h"
42 #include "DNA_brush_types.h"
43 #include "DNA_space_types.h"
44
45 #include "BLI_bitmap.h"
46 #include "BLI_utildefines.h"
47 #include "BLI_math_vector.h"
48 #include "BLI_listbase.h"
49
50 #include "BKE_brush.h"
51 #include "BKE_main.h"
52 #include "BKE_context.h"
53 #include "BKE_crazyspace.h"
54 #include "BKE_depsgraph.h"
55 #include "BKE_global.h"
56 #include "BKE_image.h"
57 #include "BKE_key.h"
58 #include "BKE_library.h"
59 #include "BKE_mesh.h"
60 #include "BKE_modifier.h"
61 #include "BKE_object.h"
62 #include "BKE_paint.h"
63 #include "BKE_pbvh.h"
64 #include "BKE_subsurf.h"
65
66 #include "bmesh.h"
67
68 const char PAINT_CURSOR_SCULPT[3] = {255, 100, 100};
69 const char PAINT_CURSOR_VERTEX_PAINT[3] = {255, 255, 255};
70 const char PAINT_CURSOR_WEIGHT_PAINT[3] = {200, 200, 255};
71 const char PAINT_CURSOR_TEXTURE_PAINT[3] = {255, 255, 255};
72
73 static OverlayControlFlags overlay_flags = 0;
74
75 void BKE_paint_invalidate_overlay_tex(Scene *scene, const Tex *tex)
76 {
77         Paint *p = BKE_paint_get_active(scene);
78         Brush *br = p->brush;
79
80         if (!br)
81                 return;
82
83         if (br->mtex.tex == tex)
84                 overlay_flags |= PAINT_INVALID_OVERLAY_TEXTURE_PRIMARY;
85         if (br->mask_mtex.tex == tex)
86                 overlay_flags |= PAINT_INVALID_OVERLAY_TEXTURE_SECONDARY;
87 }
88
89 void BKE_paint_invalidate_cursor_overlay(Scene *scene, CurveMapping *curve)
90 {
91         Paint *p = BKE_paint_get_active(scene);
92         Brush *br = p->brush;
93
94         if (br && br->curve == curve)
95                 overlay_flags |= PAINT_INVALID_OVERLAY_CURVE;
96 }
97
98 void BKE_paint_invalidate_overlay_all(void)
99 {
100         overlay_flags |= (PAINT_INVALID_OVERLAY_TEXTURE_SECONDARY |
101                           PAINT_INVALID_OVERLAY_TEXTURE_PRIMARY |
102                           PAINT_INVALID_OVERLAY_CURVE);
103 }
104
105 OverlayControlFlags BKE_paint_get_overlay_flags(void)
106 {
107         return overlay_flags;
108 }
109
110 void BKE_paint_set_overlay_override(OverlayFlags flags)
111 {
112         if (flags & BRUSH_OVERLAY_OVERRIDE_MASK) {
113                 if (flags & BRUSH_OVERLAY_CURSOR_OVERRIDE_ON_STROKE)
114                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_CURSOR;
115                 if (flags & BRUSH_OVERLAY_PRIMARY_OVERRIDE_ON_STROKE)
116                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_PRIMARY;
117                 if (flags & BRUSH_OVERLAY_SECONDARY_OVERRIDE_ON_STROKE)
118                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_SECONDARY;
119         }
120         else {
121                 overlay_flags &= ~(PAINT_OVERRIDE_MASK);
122         }
123 }
124
125 void BKE_paint_reset_overlay_invalid(OverlayControlFlags flag)
126 {
127         overlay_flags &= ~(flag);
128 }
129
130
131 Paint *BKE_paint_get_active(Scene *sce)
132 {
133         if (sce) {
134                 ToolSettings *ts = sce->toolsettings;
135                 
136                 if (sce->basact && sce->basact->object) {
137                         switch (sce->basact->object->mode) {
138                                 case OB_MODE_SCULPT:
139                                         return &ts->sculpt->paint;
140                                 case OB_MODE_VERTEX_PAINT:
141                                         return &ts->vpaint->paint;
142                                 case OB_MODE_WEIGHT_PAINT:
143                                         return &ts->wpaint->paint;
144                                 case OB_MODE_TEXTURE_PAINT:
145                                         return &ts->imapaint.paint;
146                                 case OB_MODE_EDIT:
147                                         if (ts->use_uv_sculpt)
148                                                 return &ts->uvsculpt->paint;
149                                         return &ts->imapaint.paint;
150                         }
151                 }
152
153                 /* default to image paint */
154                 return &ts->imapaint.paint;
155         }
156
157         return NULL;
158 }
159
160 Paint *BKE_paint_get_active_from_context(const bContext *C)
161 {
162         Scene *sce = CTX_data_scene(C);
163         SpaceImage *sima;
164
165         if (sce) {
166                 ToolSettings *ts = sce->toolsettings;
167                 Object *obact = NULL;
168
169                 if (sce->basact && sce->basact->object)
170                         obact = sce->basact->object;
171
172                 if ((sima = CTX_wm_space_image(C)) != NULL) {
173                         if (obact && obact->mode == OB_MODE_EDIT) {
174                                 if (sima->mode == SI_MODE_PAINT)
175                                         return &ts->imapaint.paint;
176                                 else if (ts->use_uv_sculpt)
177                                         return &ts->uvsculpt->paint;
178                         }
179                         else {
180                                 return &ts->imapaint.paint;
181                         }
182                 }
183                 else if (obact) {
184                         switch (obact->mode) {
185                                 case OB_MODE_SCULPT:
186                                         return &ts->sculpt->paint;
187                                 case OB_MODE_VERTEX_PAINT:
188                                         return &ts->vpaint->paint;
189                                 case OB_MODE_WEIGHT_PAINT:
190                                         return &ts->wpaint->paint;
191                                 case OB_MODE_TEXTURE_PAINT:
192                                         return &ts->imapaint.paint;
193                                 case OB_MODE_EDIT:
194                                         if (ts->use_uv_sculpt)
195                                                 return &ts->uvsculpt->paint;
196                                         return &ts->imapaint.paint;
197                                 default:
198                                         return &ts->imapaint.paint;
199                         }
200                 }
201                 else {
202                         /* default to image paint */
203                         return &ts->imapaint.paint;
204                 }
205         }
206
207         return NULL;
208 }
209
210 PaintMode BKE_paintmode_get_active_from_context(const bContext *C)
211 {
212         Scene *sce = CTX_data_scene(C);
213         SpaceImage *sima;
214
215         if (sce) {
216                 ToolSettings *ts = sce->toolsettings;
217                 Object *obact = NULL;
218
219                 if (sce->basact && sce->basact->object)
220                         obact = sce->basact->object;
221
222                 if ((sima = CTX_wm_space_image(C)) != NULL) {
223                         if (obact && obact->mode == OB_MODE_EDIT) {
224                                 if (sima->mode == SI_MODE_PAINT)
225                                         return PAINT_TEXTURE_2D;
226                                 else if (ts->use_uv_sculpt)
227                                         return PAINT_SCULPT_UV;
228                         }
229                         else {
230                                 return PAINT_TEXTURE_2D;
231                         }
232                 }
233                 else if (obact) {
234                         switch (obact->mode) {
235                                 case OB_MODE_SCULPT:
236                                         return PAINT_SCULPT;
237                                 case OB_MODE_VERTEX_PAINT:
238                                         return PAINT_VERTEX;
239                                 case OB_MODE_WEIGHT_PAINT:
240                                         return PAINT_WEIGHT;
241                                 case OB_MODE_TEXTURE_PAINT:
242                                         return PAINT_TEXTURE_PROJECTIVE;
243                                 case OB_MODE_EDIT:
244                                         if (ts->use_uv_sculpt)
245                                                 return PAINT_SCULPT_UV;
246                                         return PAINT_TEXTURE_2D;
247                                 default:
248                                         return PAINT_TEXTURE_2D;
249                         }
250                 }
251                 else {
252                         /* default to image paint */
253                         return PAINT_TEXTURE_2D;
254                 }
255         }
256
257         return PAINT_INVALID;
258 }
259
260 Brush *BKE_paint_brush(Paint *p)
261 {
262         return p ? p->brush : NULL;
263 }
264
265 void BKE_paint_brush_set(Paint *p, Brush *br)
266 {
267         if (p) {
268                 id_us_min((ID *)p->brush);
269                 id_us_plus((ID *)br);
270                 p->brush = br;
271         }
272 }
273
274 void BKE_paint_curve_free(PaintCurve *pc)
275 {
276         if (pc->points) {
277                 MEM_freeN(pc->points);
278                 pc->points = NULL;
279                 pc->tot_points = 0;
280         }
281 }
282
283 PaintCurve *BKE_paint_curve_add(Main *bmain, const char *name)
284 {
285         PaintCurve *pc;
286
287         pc = BKE_libblock_alloc(bmain, ID_PC, name);
288
289         return pc;
290 }
291
292 Palette *BKE_paint_palette(Paint *p)
293 {
294         return p ? p->palette : NULL;
295 }
296
297 void BKE_paint_palette_set(Paint *p, Palette *palette)
298 {
299         if (p) {
300                 id_us_min((ID *)p->palette);
301                 id_us_plus((ID *)palette);
302                 p->palette = palette;
303         }
304 }
305
306 void BKE_paint_curve_set(Brush *br, PaintCurve *pc)
307 {
308         if (br) {
309                 id_us_min((ID *)br->paint_curve);
310                 id_us_plus((ID *)pc);
311                 br->paint_curve = pc;
312         }
313 }
314
315 /* remove colour from palette. Must be certain color is inside the palette! */
316 void BKE_palette_color_remove(Palette *palette, PaletteColor *color)
317 {
318         BLI_remlink(&palette->colors, color);
319         BLI_addhead(&palette->deleted, color);
320 }
321
322 void BKE_palette_cleanup(Palette *palette)
323 {
324         BLI_freelistN(&palette->deleted);
325 }
326
327
328 Palette *BKE_palette_add(Main *bmain, const char *name)
329 {
330         Palette *palette;
331
332         palette = BKE_libblock_alloc(bmain, ID_PAL, name);
333
334         /* enable fake user by default */
335         palette->id.flag |= LIB_FAKEUSER;
336
337         return palette;
338 }
339
340 void BKE_palette_free(Palette *palette)
341 {
342         BLI_freelistN(&palette->colors);
343 }
344
345 PaletteColor *BKE_palette_color_add(Palette *palette)
346 {
347         PaletteColor *color = MEM_callocN(sizeof(*color), "Pallete Color");
348         BLI_addtail(&palette->colors, color);
349         palette->active_color = BLI_countlist(&palette->colors) - 1;
350         return color;
351 }
352
353 void BKE_palette_color_delete(struct Palette *palette)
354 {
355         PaletteColor *color = BLI_findlink(&palette->colors, palette->active_color);
356
357         if (color) {
358                 if ((color == palette->colors.last) && (palette->colors.last != palette->colors.first))
359                         palette->active_color--;
360
361                 BLI_remlink(&palette->colors, color);
362                 BLI_addhead(&palette->deleted, color);
363         }
364 }
365
366
367 bool BKE_palette_is_empty(const struct Palette *palette)
368 {
369         return BLI_listbase_is_empty(&palette->colors);
370 }
371
372
373 /* are we in vertex paint or weight pain face select mode? */
374 bool BKE_paint_select_face_test(Object *ob)
375 {
376         return ( (ob != NULL) &&
377                  (ob->type == OB_MESH) &&
378                  (ob->data != NULL) &&
379                  (((Mesh *)ob->data)->editflag & ME_EDIT_PAINT_FACE_SEL) &&
380                  (ob->mode & (OB_MODE_VERTEX_PAINT | OB_MODE_WEIGHT_PAINT | OB_MODE_TEXTURE_PAINT))
381                  );
382 }
383
384 /* are we in weight paint vertex select mode? */
385 bool BKE_paint_select_vert_test(Object *ob)
386 {
387         return ( (ob != NULL) &&
388                  (ob->type == OB_MESH) &&
389                  (ob->data != NULL) &&
390                  (((Mesh *)ob->data)->editflag & ME_EDIT_PAINT_VERT_SEL) &&
391                  (ob->mode & OB_MODE_WEIGHT_PAINT)
392                  );
393 }
394
395 /**
396  * used to check if selection is possible
397  * (when we don't care if its face or vert)
398  */
399 bool BKE_paint_select_elem_test(Object *ob)
400 {
401         return (BKE_paint_select_vert_test(ob) ||
402                 BKE_paint_select_face_test(ob));
403 }
404
405 void BKE_paint_init(Paint *p, const char col[3])
406 {
407         Brush *brush;
408
409         /* If there's no brush, create one */
410         brush = BKE_paint_brush(p);
411         if (brush == NULL)
412                 brush = BKE_brush_add(G.main, "Brush");
413         BKE_paint_brush_set(p, brush);
414
415         memcpy(p->paint_cursor_col, col, 3);
416         p->paint_cursor_col[3] = 128;
417 }
418
419 void BKE_paint_free(Paint *paint)
420 {
421         id_us_min((ID *)paint->brush);
422         id_us_min((ID *)paint->palette);
423 }
424
425 /* called when copying scene settings, so even if 'src' and 'tar' are the same
426  * still do a id_us_plus(), rather then if we were copying between 2 existing
427  * scenes where a matching value should decrease the existing user count as
428  * with paint_brush_set() */
429 void BKE_paint_copy(Paint *src, Paint *tar)
430 {
431         tar->brush = src->brush;
432         id_us_plus((ID *)tar->brush);
433         id_us_plus((ID *)tar->palette);
434 }
435
436 /* returns non-zero if any of the face's vertices
437  * are hidden, zero otherwise */
438 bool paint_is_face_hidden(const MFace *f, const MVert *mvert)
439 {
440         return ((mvert[f->v1].flag & ME_HIDE) ||
441                 (mvert[f->v2].flag & ME_HIDE) ||
442                 (mvert[f->v3].flag & ME_HIDE) ||
443                 (f->v4 && (mvert[f->v4].flag & ME_HIDE)));
444 }
445
446 /* returns non-zero if any of the corners of the grid
447  * face whose inner corner is at (x, y) are hidden,
448  * zero otherwise */
449 bool paint_is_grid_face_hidden(const unsigned int *grid_hidden,
450                               int gridsize, int x, int y)
451 {
452         /* skip face if any of its corners are hidden */
453         return (BLI_BITMAP_TEST(grid_hidden, y * gridsize + x) ||
454                 BLI_BITMAP_TEST(grid_hidden, y * gridsize + x + 1) ||
455                 BLI_BITMAP_TEST(grid_hidden, (y + 1) * gridsize + x + 1) ||
456                 BLI_BITMAP_TEST(grid_hidden, (y + 1) * gridsize + x));
457 }
458
459 /* Return true if all vertices in the face are visible, false otherwise */
460 bool paint_is_bmesh_face_hidden(BMFace *f)
461 {
462         BMLoop *l_iter;
463         BMLoop *l_first;
464
465         l_iter = l_first = BM_FACE_FIRST_LOOP(f);
466         do {
467                 if (BM_elem_flag_test(l_iter->v, BM_ELEM_HIDDEN)) {
468                         return true;
469                 }
470         } while ((l_iter = l_iter->next) != l_first);
471
472         return false;
473 }
474
475 float paint_grid_paint_mask(const GridPaintMask *gpm, unsigned level,
476                             unsigned x, unsigned y)
477 {
478         int factor = BKE_ccg_factor(level, gpm->level);
479         int gridsize = BKE_ccg_gridsize(gpm->level);
480         
481         return gpm->data[(y * factor) * gridsize + (x * factor)];
482 }
483
484 /* threshold to move before updating the brush rotation */
485 #define RAKE_THRESHHOLD 20
486
487 void paint_calculate_rake_rotation(UnifiedPaintSettings *ups, const float mouse_pos[2])
488 {
489         const float u = 0.5f;
490         const float r = RAKE_THRESHHOLD;
491
492         float dpos[2];
493         sub_v2_v2v2(dpos, ups->last_rake, mouse_pos);
494
495         if (len_squared_v2(dpos) >= r * r) {
496                 ups->brush_rotation = atan2(dpos[0], dpos[1]);
497
498                 interp_v2_v2v2(ups->last_rake, ups->last_rake,
499                                mouse_pos, u);
500         }
501 }
502
503 void BKE_free_sculptsession_deformMats(SculptSession *ss)
504 {
505         if (ss->orig_cos) MEM_freeN(ss->orig_cos);
506         if (ss->deform_cos) MEM_freeN(ss->deform_cos);
507         if (ss->deform_imats) MEM_freeN(ss->deform_imats);
508
509         ss->orig_cos = NULL;
510         ss->deform_cos = NULL;
511         ss->deform_imats = NULL;
512 }
513
514 /* Write out the sculpt dynamic-topology BMesh to the Mesh */
515 static void sculptsession_bm_to_me_update_data_only(Object *ob, bool reorder)
516 {
517         SculptSession *ss = ob->sculpt;
518
519         if (ss->bm) {
520                 if (ob->data) {
521                         BMIter iter;
522                         BMFace *efa;
523                         BM_ITER_MESH (efa, &iter, ss->bm, BM_FACES_OF_MESH) {
524                                 BM_elem_flag_set(efa, BM_ELEM_SMOOTH, ss->bm_smooth_shading);
525                         }
526                         if (reorder)
527                                 BM_log_mesh_elems_reorder(ss->bm, ss->bm_log);
528                         BM_mesh_bm_to_me(ss->bm, ob->data, false);
529                 }
530         }
531 }
532
533 void BKE_sculptsession_bm_to_me(Object *ob, bool reorder)
534 {
535         if (ob && ob->sculpt) {
536                 sculptsession_bm_to_me_update_data_only(ob, reorder);
537
538                 /* ensure the objects DerivedMesh mesh doesn't hold onto arrays now realloc'd in the mesh [#34473] */
539                 DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
540         }
541 }
542
543 void BKE_sculptsession_bm_to_me_for_render(Object *object)
544 {
545         if (object && object->sculpt) {
546                 if (object->sculpt->bm) {
547                         /* Ensure no points to old arrays are stored in DM
548                          *
549                          * Apparently, we could not use DAG_id_tag_update
550                          * here because this will lead to the while object
551                          * surface to disappear, so we'll release DM in place.
552                          */
553                         BKE_object_free_derived_caches(object);
554
555                         if (object->sculpt->pbvh) {
556                                 BKE_pbvh_free(object->sculpt->pbvh);
557                                 object->sculpt->pbvh = NULL;
558                         }
559
560                         sculptsession_bm_to_me_update_data_only(object, false);
561
562                         /* In contrast with sculptsession_bm_to_me no need in
563                          * DAG tag update here - derived mesh was freed and
564                          * old pointers are nowhere stored.
565                          */
566                 }
567         }
568 }
569
570 void BKE_free_sculptsession(Object *ob)
571 {
572         if (ob && ob->sculpt) {
573                 SculptSession *ss = ob->sculpt;
574                 DerivedMesh *dm = ob->derivedFinal;
575
576                 if (ss->bm) {
577                         BKE_sculptsession_bm_to_me(ob, true);
578                         BM_mesh_free(ss->bm);
579                 }
580
581                 if (ss->pbvh)
582                         BKE_pbvh_free(ss->pbvh);
583                 if (ss->bm_log)
584                         BM_log_free(ss->bm_log);
585
586                 if (dm && dm->getPBVH)
587                         dm->getPBVH(NULL, dm);  /* signal to clear */
588
589                 if (ss->texcache)
590                         MEM_freeN(ss->texcache);
591
592                 if (ss->tex_pool)
593                         BKE_image_pool_free(ss->tex_pool);
594
595                 if (ss->layer_co)
596                         MEM_freeN(ss->layer_co);
597
598                 if (ss->orig_cos)
599                         MEM_freeN(ss->orig_cos);
600                 if (ss->deform_cos)
601                         MEM_freeN(ss->deform_cos);
602                 if (ss->deform_imats)
603                         MEM_freeN(ss->deform_imats);
604
605                 MEM_freeN(ss);
606
607                 ob->sculpt = NULL;
608         }
609 }
610
611 /* Sculpt mode handles multires differently from regular meshes, but only if
612  * it's the last modifier on the stack and it is not on the first level */
613 MultiresModifierData *BKE_sculpt_multires_active(Scene *scene, Object *ob)
614 {
615         Mesh *me = (Mesh *)ob->data;
616         ModifierData *md;
617         VirtualModifierData virtualModifierData;
618
619         if (ob->sculpt && ob->sculpt->bm) {
620                 /* can't combine multires and dynamic topology */
621                 return NULL;
622         }
623
624         if (!CustomData_get_layer(&me->ldata, CD_MDISPS)) {
625                 /* multires can't work without displacement layer */
626                 return NULL;
627         }
628
629         for (md = modifiers_getVirtualModifierList(ob, &virtualModifierData); md; md = md->next) {
630                 if (md->type == eModifierType_Multires) {
631                         MultiresModifierData *mmd = (MultiresModifierData *)md;
632
633                         if (!modifier_isEnabled(scene, md, eModifierMode_Realtime))
634                                 continue;
635
636                         if (mmd->sculptlvl > 0) return mmd;
637                         else return NULL;
638                 }
639         }
640
641         return NULL;
642 }
643
644
645 /* Checks if there are any supported deformation modifiers active */
646 static bool sculpt_modifiers_active(Scene *scene, Sculpt *sd, Object *ob)
647 {
648         ModifierData *md;
649         Mesh *me = (Mesh *)ob->data;
650         MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
651         VirtualModifierData virtualModifierData;
652
653         if (mmd || ob->sculpt->bm)
654                 return 0;
655
656         /* non-locked shape keys could be handled in the same way as deformed mesh */
657         if ((ob->shapeflag & OB_SHAPE_LOCK) == 0 && me->key && ob->shapenr)
658                 return 1;
659
660         md = modifiers_getVirtualModifierList(ob, &virtualModifierData);
661
662         /* exception for shape keys because we can edit those */
663         for (; md; md = md->next) {
664                 ModifierTypeInfo *mti = modifierType_getInfo(md->type);
665                 if (!modifier_isEnabled(scene, md, eModifierMode_Realtime)) continue;
666                 if (ELEM(md->type, eModifierType_ShapeKey, eModifierType_Multires)) continue;
667
668                 if (mti->type == eModifierTypeType_OnlyDeform) return 1;
669                 else if ((sd->flags & SCULPT_ONLY_DEFORM) == 0) return 1;
670         }
671
672         return 0;
673 }
674
675 /**
676  * \param need_mask So the DerivedMesh thats returned has mask data
677  */
678 void BKE_sculpt_update_mesh_elements(Scene *scene, Sculpt *sd, Object *ob,
679                                      bool need_pmap, bool need_mask)
680 {
681         DerivedMesh *dm;
682         SculptSession *ss = ob->sculpt;
683         Mesh *me = ob->data;
684         MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
685
686         ss->modifiers_active = sculpt_modifiers_active(scene, sd, ob);
687         ss->show_diffuse_color = (sd->flags & SCULPT_SHOW_DIFFUSE) != 0;
688
689         if (need_mask) {
690                 if (mmd == NULL) {
691                         if (!CustomData_has_layer(&me->vdata, CD_PAINT_MASK)) {
692                                 BKE_sculpt_mask_layers_ensure(ob, NULL);
693                         }
694                 }
695                 else {
696                         if (!CustomData_has_layer(&me->ldata, CD_GRID_PAINT_MASK)) {
697 #if 1
698                                 BKE_sculpt_mask_layers_ensure(ob, mmd);
699 #else                           /* if we wanted to support adding mask data while multi-res painting, we would need to do this */
700                                 if ((ED_sculpt_mask_layers_ensure(ob, mmd) & ED_SCULPT_MASK_LAYER_CALC_LOOP)) {
701                                         /* remake the derived mesh */
702                                         ob->recalc |= OB_RECALC_DATA;
703                                         BKE_object_handle_update(scene, ob);
704                                 }
705 #endif
706                         }
707                 }
708         }
709
710         /* BMESH ONLY --- at some point we should move sculpt code to use polygons only - but for now it needs tessfaces */
711         BKE_mesh_tessface_ensure(me);
712
713         if (!mmd) ss->kb = BKE_keyblock_from_object(ob);
714         else ss->kb = NULL;
715
716         /* needs to be called after we ensure tessface */
717         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
718
719         if (mmd) {
720                 ss->multires = mmd;
721                 ss->totvert = dm->getNumVerts(dm);
722                 ss->totpoly = dm->getNumPolys(dm);
723                 ss->mvert = NULL;
724                 ss->mpoly = NULL;
725                 ss->mloop = NULL;
726                 ss->face_normals = NULL;
727         }
728         else {
729                 ss->totvert = me->totvert;
730                 ss->totpoly = me->totpoly;
731                 ss->mvert = me->mvert;
732                 ss->mpoly = me->mpoly;
733                 ss->mloop = me->mloop;
734                 ss->face_normals = NULL;
735                 ss->multires = NULL;
736                 ss->vmask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
737         }
738
739         ss->pbvh = dm->getPBVH(ob, dm);
740         ss->pmap = (need_pmap && dm->getPolyMap) ? dm->getPolyMap(ob, dm) : NULL;
741
742         pbvh_show_diffuse_color_set(ss->pbvh, ss->show_diffuse_color);
743
744         if (ss->modifiers_active) {
745                 if (!ss->orig_cos) {
746                         int a;
747
748                         BKE_free_sculptsession_deformMats(ss);
749
750                         ss->orig_cos = (ss->kb) ? BKE_key_convert_to_vertcos(ob, ss->kb) : BKE_mesh_vertexCos_get(me, NULL);
751
752                         BKE_crazyspace_build_sculpt(scene, ob, &ss->deform_imats, &ss->deform_cos);
753                         BKE_pbvh_apply_vertCos(ss->pbvh, ss->deform_cos);
754
755                         for (a = 0; a < me->totvert; ++a) {
756                                 invert_m3(ss->deform_imats[a]);
757                         }
758                 }
759         }
760         else {
761                 BKE_free_sculptsession_deformMats(ss);
762         }
763
764         if (ss->kb != NULL && ss->deform_cos == NULL) {
765                 ss->deform_cos = BKE_key_convert_to_vertcos(ob, ss->kb);
766         }
767
768         /* if pbvh is deformed, key block is already applied to it */
769         if (ss->kb) {
770                 bool pbvh_deformd = BKE_pbvh_isDeformed(ss->pbvh);
771                 if (!pbvh_deformd || ss->deform_cos == NULL) {
772                         float (*vertCos)[3] = BKE_key_convert_to_vertcos(ob, ss->kb);
773
774                         if (vertCos) {
775                                 if (!pbvh_deformd) {
776                                         /* apply shape keys coordinates to PBVH */
777                                         BKE_pbvh_apply_vertCos(ss->pbvh, vertCos);
778                                 }
779                                 if (ss->deform_cos == NULL) {
780                                         ss->deform_cos = vertCos;
781                                 }
782                                 if (vertCos != ss->deform_cos) {
783                                         MEM_freeN(vertCos);
784                                 }
785                         }
786                 }
787         }
788 }
789
790 int BKE_sculpt_mask_layers_ensure(Object *ob, MultiresModifierData *mmd)
791 {
792         const float *paint_mask;
793         Mesh *me = ob->data;
794         int ret = 0;
795
796         paint_mask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
797
798         /* if multires is active, create a grid paint mask layer if there
799          * isn't one already */
800         if (mmd && !CustomData_has_layer(&me->ldata, CD_GRID_PAINT_MASK)) {
801                 GridPaintMask *gmask;
802                 int level = max_ii(1, mmd->sculptlvl);
803                 int gridsize = BKE_ccg_gridsize(level);
804                 int gridarea = gridsize * gridsize;
805                 int i, j;
806
807                 gmask = CustomData_add_layer(&me->ldata, CD_GRID_PAINT_MASK,
808                                              CD_CALLOC, NULL, me->totloop);
809
810                 for (i = 0; i < me->totloop; i++) {
811                         GridPaintMask *gpm = &gmask[i];
812
813                         gpm->level = level;
814                         gpm->data = MEM_callocN(sizeof(float) * gridarea,
815                                                 "GridPaintMask.data");
816                 }
817
818                 /* if vertices already have mask, copy into multires data */
819                 if (paint_mask) {
820                         for (i = 0; i < me->totpoly; i++) {
821                                 const MPoly *p = &me->mpoly[i];
822                                 float avg = 0;
823
824                                 /* mask center */
825                                 for (j = 0; j < p->totloop; j++) {
826                                         const MLoop *l = &me->mloop[p->loopstart + j];
827                                         avg += paint_mask[l->v];
828                                 }
829                                 avg /= (float)p->totloop;
830
831                                 /* fill in multires mask corner */
832                                 for (j = 0; j < p->totloop; j++) {
833                                         GridPaintMask *gpm = &gmask[p->loopstart + j];
834                                         const MLoop *l = &me->mloop[p->loopstart + j];
835                                         const MLoop *prev = ME_POLY_LOOP_PREV(me->mloop, p, j);
836                                         const MLoop *next = ME_POLY_LOOP_NEXT(me->mloop, p, j);
837
838                                         gpm->data[0] = avg;
839                                         gpm->data[1] = (paint_mask[l->v] +
840                                                         paint_mask[next->v]) * 0.5f;
841                                         gpm->data[2] = (paint_mask[l->v] +
842                                                         paint_mask[prev->v]) * 0.5f;
843                                         gpm->data[3] = paint_mask[l->v];
844                                 }
845                         }
846                 }
847
848                 ret |= SCULPT_MASK_LAYER_CALC_LOOP;
849         }
850
851         /* create vertex paint mask layer if there isn't one already */
852         if (!paint_mask) {
853                 CustomData_add_layer(&me->vdata, CD_PAINT_MASK,
854                                      CD_CALLOC, NULL, me->totvert);
855                 ret |= SCULPT_MASK_LAYER_CALC_VERT;
856         }
857
858         return ret;
859 }