fd5d14b6cfc75faf9790de522a4447354dbff9c4
[blender-staging.git] / source / blender / blenkernel / intern / paint.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software  Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2009 by Nicholas Bishop
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s): none yet.
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  */
27
28 /** \file blender/blenkernel/intern/paint.c
29  *  \ingroup bke
30  */
31
32 #include <stdlib.h>
33 #include <string.h>
34
35 #include "MEM_guardedalloc.h"
36
37 #include "DNA_object_types.h"
38 #include "DNA_mesh_types.h"
39 #include "DNA_meshdata_types.h"
40 #include "DNA_modifier_types.h"
41 #include "DNA_scene_types.h"
42 #include "DNA_brush_types.h"
43 #include "DNA_space_types.h"
44
45 #include "BLI_bitmap.h"
46 #include "BLI_utildefines.h"
47 #include "BLI_math_vector.h"
48 #include "BLI_listbase.h"
49
50 #include "BKE_brush.h"
51 #include "BKE_colortools.h"
52 #include "BKE_main.h"
53 #include "BKE_context.h"
54 #include "BKE_crazyspace.h"
55 #include "BKE_depsgraph.h"
56 #include "BKE_global.h"
57 #include "BKE_image.h"
58 #include "BKE_key.h"
59 #include "BKE_library.h"
60 #include "BKE_mesh.h"
61 #include "BKE_modifier.h"
62 #include "BKE_object.h"
63 #include "BKE_paint.h"
64 #include "BKE_pbvh.h"
65 #include "BKE_subsurf.h"
66
67 #include "bmesh.h"
68
69 const char PAINT_CURSOR_SCULPT[3] = {255, 100, 100};
70 const char PAINT_CURSOR_VERTEX_PAINT[3] = {255, 255, 255};
71 const char PAINT_CURSOR_WEIGHT_PAINT[3] = {200, 200, 255};
72 const char PAINT_CURSOR_TEXTURE_PAINT[3] = {255, 255, 255};
73
74 static OverlayControlFlags overlay_flags = 0;
75
76 void BKE_paint_invalidate_overlay_tex(Scene *scene, const Tex *tex)
77 {
78         Paint *p = BKE_paint_get_active(scene);
79         Brush *br = p->brush;
80
81         if (!br)
82                 return;
83
84         if (br->mtex.tex == tex)
85                 overlay_flags |= PAINT_INVALID_OVERLAY_TEXTURE_PRIMARY;
86         if (br->mask_mtex.tex == tex)
87                 overlay_flags |= PAINT_INVALID_OVERLAY_TEXTURE_SECONDARY;
88 }
89
90 void BKE_paint_invalidate_cursor_overlay(Scene *scene, CurveMapping *curve)
91 {
92         Paint *p = BKE_paint_get_active(scene);
93         Brush *br = p->brush;
94
95         if (br && br->curve == curve)
96                 overlay_flags |= PAINT_INVALID_OVERLAY_CURVE;
97 }
98
99 void BKE_paint_invalidate_overlay_all(void)
100 {
101         overlay_flags |= (PAINT_INVALID_OVERLAY_TEXTURE_SECONDARY |
102                           PAINT_INVALID_OVERLAY_TEXTURE_PRIMARY |
103                           PAINT_INVALID_OVERLAY_CURVE);
104 }
105
106 OverlayControlFlags BKE_paint_get_overlay_flags(void)
107 {
108         return overlay_flags;
109 }
110
111 void BKE_paint_set_overlay_override(OverlayFlags flags)
112 {
113         if (flags & BRUSH_OVERLAY_OVERRIDE_MASK) {
114                 if (flags & BRUSH_OVERLAY_CURSOR_OVERRIDE_ON_STROKE)
115                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_CURSOR;
116                 if (flags & BRUSH_OVERLAY_PRIMARY_OVERRIDE_ON_STROKE)
117                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_PRIMARY;
118                 if (flags & BRUSH_OVERLAY_SECONDARY_OVERRIDE_ON_STROKE)
119                         overlay_flags |= PAINT_OVERLAY_OVERRIDE_SECONDARY;
120         }
121         else {
122                 overlay_flags &= ~(PAINT_OVERRIDE_MASK);
123         }
124 }
125
126 void BKE_paint_reset_overlay_invalid(OverlayControlFlags flag)
127 {
128         overlay_flags &= ~(flag);
129 }
130
131
132 Paint *BKE_paint_get_active(Scene *sce)
133 {
134         if (sce) {
135                 ToolSettings *ts = sce->toolsettings;
136                 
137                 if (sce->basact && sce->basact->object) {
138                         switch (sce->basact->object->mode) {
139                                 case OB_MODE_SCULPT:
140                                         return &ts->sculpt->paint;
141                                 case OB_MODE_VERTEX_PAINT:
142                                         return &ts->vpaint->paint;
143                                 case OB_MODE_WEIGHT_PAINT:
144                                         return &ts->wpaint->paint;
145                                 case OB_MODE_TEXTURE_PAINT:
146                                         return &ts->imapaint.paint;
147                                 case OB_MODE_EDIT:
148                                         if (ts->use_uv_sculpt)
149                                                 return &ts->uvsculpt->paint;
150                                         return &ts->imapaint.paint;
151                         }
152                 }
153
154                 /* default to image paint */
155                 return &ts->imapaint.paint;
156         }
157
158         return NULL;
159 }
160
161 Paint *BKE_paint_get_active_from_context(const bContext *C)
162 {
163         Scene *sce = CTX_data_scene(C);
164         SpaceImage *sima;
165
166         if (sce) {
167                 ToolSettings *ts = sce->toolsettings;
168                 Object *obact = NULL;
169
170                 if (sce->basact && sce->basact->object)
171                         obact = sce->basact->object;
172
173                 if ((sima = CTX_wm_space_image(C)) != NULL) {
174                         if (obact && obact->mode == OB_MODE_EDIT) {
175                                 if (sima->mode == SI_MODE_PAINT)
176                                         return &ts->imapaint.paint;
177                                 else if (ts->use_uv_sculpt)
178                                         return &ts->uvsculpt->paint;
179                         }
180                         else {
181                                 return &ts->imapaint.paint;
182                         }
183                 }
184                 else if (obact) {
185                         switch (obact->mode) {
186                                 case OB_MODE_SCULPT:
187                                         return &ts->sculpt->paint;
188                                 case OB_MODE_VERTEX_PAINT:
189                                         return &ts->vpaint->paint;
190                                 case OB_MODE_WEIGHT_PAINT:
191                                         return &ts->wpaint->paint;
192                                 case OB_MODE_TEXTURE_PAINT:
193                                         return &ts->imapaint.paint;
194                                 case OB_MODE_EDIT:
195                                         if (ts->use_uv_sculpt)
196                                                 return &ts->uvsculpt->paint;
197                                         return &ts->imapaint.paint;
198                                 default:
199                                         return &ts->imapaint.paint;
200                         }
201                 }
202                 else {
203                         /* default to image paint */
204                         return &ts->imapaint.paint;
205                 }
206         }
207
208         return NULL;
209 }
210
211 PaintMode BKE_paintmode_get_active_from_context(const bContext *C)
212 {
213         Scene *sce = CTX_data_scene(C);
214         SpaceImage *sima;
215
216         if (sce) {
217                 ToolSettings *ts = sce->toolsettings;
218                 Object *obact = NULL;
219
220                 if (sce->basact && sce->basact->object)
221                         obact = sce->basact->object;
222
223                 if ((sima = CTX_wm_space_image(C)) != NULL) {
224                         if (obact && obact->mode == OB_MODE_EDIT) {
225                                 if (sima->mode == SI_MODE_PAINT)
226                                         return PAINT_TEXTURE_2D;
227                                 else if (ts->use_uv_sculpt)
228                                         return PAINT_SCULPT_UV;
229                         }
230                         else {
231                                 return PAINT_TEXTURE_2D;
232                         }
233                 }
234                 else if (obact) {
235                         switch (obact->mode) {
236                                 case OB_MODE_SCULPT:
237                                         return PAINT_SCULPT;
238                                 case OB_MODE_VERTEX_PAINT:
239                                         return PAINT_VERTEX;
240                                 case OB_MODE_WEIGHT_PAINT:
241                                         return PAINT_WEIGHT;
242                                 case OB_MODE_TEXTURE_PAINT:
243                                         return PAINT_TEXTURE_PROJECTIVE;
244                                 case OB_MODE_EDIT:
245                                         if (ts->use_uv_sculpt)
246                                                 return PAINT_SCULPT_UV;
247                                         return PAINT_TEXTURE_2D;
248                                 default:
249                                         return PAINT_TEXTURE_2D;
250                         }
251                 }
252                 else {
253                         /* default to image paint */
254                         return PAINT_TEXTURE_2D;
255                 }
256         }
257
258         return PAINT_INVALID;
259 }
260
261 Brush *BKE_paint_brush(Paint *p)
262 {
263         return p ? p->brush : NULL;
264 }
265
266 void BKE_paint_brush_set(Paint *p, Brush *br)
267 {
268         if (p) {
269                 id_us_min((ID *)p->brush);
270                 id_us_plus((ID *)br);
271                 p->brush = br;
272         }
273 }
274
275 void BKE_paint_curve_free(PaintCurve *pc)
276 {
277         if (pc->points) {
278                 MEM_freeN(pc->points);
279                 pc->points = NULL;
280                 pc->tot_points = 0;
281         }
282 }
283
284 PaintCurve *BKE_paint_curve_add(Main *bmain, const char *name)
285 {
286         PaintCurve *pc;
287
288         pc = BKE_libblock_alloc(bmain, ID_PC, name);
289
290         return pc;
291 }
292
293 Palette *BKE_paint_palette(Paint *p)
294 {
295         return p ? p->palette : NULL;
296 }
297
298 void BKE_paint_palette_set(Paint *p, Palette *palette)
299 {
300         if (p) {
301                 id_us_min((ID *)p->palette);
302                 id_us_plus((ID *)palette);
303                 p->palette = palette;
304         }
305 }
306
307 void BKE_paint_curve_set(Brush *br, PaintCurve *pc)
308 {
309         if (br) {
310                 id_us_min((ID *)br->paint_curve);
311                 id_us_plus((ID *)pc);
312                 br->paint_curve = pc;
313         }
314 }
315
316 void BKE_paint_curve_clamp_endpoint_add_index(PaintCurve *pc, const int add_index)
317 {
318         pc->add_index = (add_index || pc->tot_points == 1) ? (add_index + 1) : 0;
319 }
320
321 /* remove colour from palette. Must be certain color is inside the palette! */
322 void BKE_palette_color_remove(Palette *palette, PaletteColor *color)
323 {
324         if (BLI_listbase_count_ex(&palette->colors, palette->active_color) == palette->active_color) {
325                 palette->active_color--;
326         }
327
328         BLI_remlink(&palette->colors, color);
329
330         if (palette->active_color < 0 && !BLI_listbase_is_empty(&palette->colors)) {
331                 palette->active_color = 0;
332         }
333
334         MEM_freeN(color);
335 }
336
337 void BKE_palette_clear(Palette *palette)
338 {
339         BLI_freelistN(&palette->colors);
340         palette->active_color = 0;
341 }
342
343 Palette *BKE_palette_add(Main *bmain, const char *name)
344 {
345         Palette *palette;
346
347         palette = BKE_libblock_alloc(bmain, ID_PAL, name);
348
349         /* enable fake user by default */
350         palette->id.flag |= LIB_FAKEUSER;
351
352         return palette;
353 }
354
355 void BKE_palette_free(Palette *palette)
356 {
357         BLI_freelistN(&palette->colors);
358 }
359
360 PaletteColor *BKE_palette_color_add(Palette *palette)
361 {
362         PaletteColor *color = MEM_callocN(sizeof(*color), "Pallete Color");
363         BLI_addtail(&palette->colors, color);
364         return color;
365 }
366
367
368 bool BKE_palette_is_empty(const struct Palette *palette)
369 {
370         return BLI_listbase_is_empty(&palette->colors);
371 }
372
373
374 /* are we in vertex paint or weight pain face select mode? */
375 bool BKE_paint_select_face_test(Object *ob)
376 {
377         return ( (ob != NULL) &&
378                  (ob->type == OB_MESH) &&
379                  (ob->data != NULL) &&
380                  (((Mesh *)ob->data)->editflag & ME_EDIT_PAINT_FACE_SEL) &&
381                  (ob->mode & (OB_MODE_VERTEX_PAINT | OB_MODE_WEIGHT_PAINT | OB_MODE_TEXTURE_PAINT))
382                  );
383 }
384
385 /* are we in weight paint vertex select mode? */
386 bool BKE_paint_select_vert_test(Object *ob)
387 {
388         return ( (ob != NULL) &&
389                  (ob->type == OB_MESH) &&
390                  (ob->data != NULL) &&
391                  (((Mesh *)ob->data)->editflag & ME_EDIT_PAINT_VERT_SEL) &&
392                  (ob->mode & OB_MODE_WEIGHT_PAINT)
393                  );
394 }
395
396 /**
397  * used to check if selection is possible
398  * (when we don't care if its face or vert)
399  */
400 bool BKE_paint_select_elem_test(Object *ob)
401 {
402         return (BKE_paint_select_vert_test(ob) ||
403                 BKE_paint_select_face_test(ob));
404 }
405
406 void BKE_paint_cavity_curve_preset(Paint *p, int preset)
407 {
408         CurveMap *cm = NULL;
409
410         if (!p->cavity_curve)
411                 p->cavity_curve = curvemapping_add(1, 0, 0, 1, 1);
412
413         cm = p->cavity_curve->cm;
414         cm->flag &= ~CUMA_EXTEND_EXTRAPOLATE;
415
416         p->cavity_curve->preset = preset;
417         curvemap_reset(cm, &p->cavity_curve->clipr, p->cavity_curve->preset, CURVEMAP_SLOPE_POSITIVE);
418         curvemapping_changed(p->cavity_curve, false);
419 }
420
421 void BKE_paint_init(UnifiedPaintSettings *ups, Paint *p, const char col[3])
422 {
423         Brush *brush;
424
425         /* If there's no brush, create one */
426         brush = BKE_paint_brush(p);
427         if (brush == NULL)
428                 brush = BKE_brush_add(G.main, "Brush");
429         BKE_paint_brush_set(p, brush);
430
431         memcpy(p->paint_cursor_col, col, 3);
432         p->paint_cursor_col[3] = 128;
433         ups->last_stroke_valid = false;
434         zero_v3(ups->average_stroke_accum);
435         ups->average_stroke_counter = 0;
436         if (!p->cavity_curve)
437                 BKE_paint_cavity_curve_preset(p, CURVE_PRESET_LINE);
438 }
439
440 void BKE_paint_free(Paint *paint)
441 {
442         id_us_min((ID *)paint->brush);
443         id_us_min((ID *)paint->palette);
444         curvemapping_free(paint->cavity_curve);
445 }
446
447 /* called when copying scene settings, so even if 'src' and 'tar' are the same
448  * still do a id_us_plus(), rather then if we were copying between 2 existing
449  * scenes where a matching value should decrease the existing user count as
450  * with paint_brush_set() */
451 void BKE_paint_copy(Paint *src, Paint *tar)
452 {
453         tar->brush = src->brush;
454         id_us_plus((ID *)tar->brush);
455         id_us_plus((ID *)tar->palette);
456         tar->cavity_curve = curvemapping_copy(src->cavity_curve);
457 }
458
459 void BKE_paint_stroke_get_average(Scene *scene, Object *ob, float stroke[3])
460 {
461         UnifiedPaintSettings *ups = &scene->toolsettings->unified_paint_settings;
462         if (ups->last_stroke_valid && ups->average_stroke_counter > 0) {
463                 float fac = 1.0f / ups->average_stroke_counter;
464                 mul_v3_v3fl(stroke, ups->average_stroke_accum, fac);
465         }
466         else {
467                 copy_v3_v3(stroke, ob->obmat[3]);
468         }
469 }
470
471 /* returns non-zero if any of the face's vertices
472  * are hidden, zero otherwise */
473 bool paint_is_face_hidden(const MFace *f, const MVert *mvert)
474 {
475         return ((mvert[f->v1].flag & ME_HIDE) ||
476                 (mvert[f->v2].flag & ME_HIDE) ||
477                 (mvert[f->v3].flag & ME_HIDE) ||
478                 (f->v4 && (mvert[f->v4].flag & ME_HIDE)));
479 }
480
481 /* returns non-zero if any of the corners of the grid
482  * face whose inner corner is at (x, y) are hidden,
483  * zero otherwise */
484 bool paint_is_grid_face_hidden(const unsigned int *grid_hidden,
485                               int gridsize, int x, int y)
486 {
487         /* skip face if any of its corners are hidden */
488         return (BLI_BITMAP_TEST(grid_hidden, y * gridsize + x) ||
489                 BLI_BITMAP_TEST(grid_hidden, y * gridsize + x + 1) ||
490                 BLI_BITMAP_TEST(grid_hidden, (y + 1) * gridsize + x + 1) ||
491                 BLI_BITMAP_TEST(grid_hidden, (y + 1) * gridsize + x));
492 }
493
494 /* Return true if all vertices in the face are visible, false otherwise */
495 bool paint_is_bmesh_face_hidden(BMFace *f)
496 {
497         BMLoop *l_iter;
498         BMLoop *l_first;
499
500         l_iter = l_first = BM_FACE_FIRST_LOOP(f);
501         do {
502                 if (BM_elem_flag_test(l_iter->v, BM_ELEM_HIDDEN)) {
503                         return true;
504                 }
505         } while ((l_iter = l_iter->next) != l_first);
506
507         return false;
508 }
509
510 float paint_grid_paint_mask(const GridPaintMask *gpm, unsigned level,
511                             unsigned x, unsigned y)
512 {
513         int factor = BKE_ccg_factor(level, gpm->level);
514         int gridsize = BKE_ccg_gridsize(gpm->level);
515         
516         return gpm->data[(y * factor) * gridsize + (x * factor)];
517 }
518
519 /* threshold to move before updating the brush rotation */
520 #define RAKE_THRESHHOLD 20
521
522 static void update_brush_rake_rotation(UnifiedPaintSettings *ups, Brush *brush, float rotation)
523 {
524         if (brush->mtex.brush_angle_mode & MTEX_ANGLE_RAKE)
525                 ups->brush_rotation = rotation;
526         else
527                 ups->brush_rotation = 0.0f;
528
529         if (brush->mask_mtex.brush_angle_mode & MTEX_ANGLE_RAKE)
530                 /* here, translation contains the mouse coordinates. */
531                 ups->brush_rotation_sec = rotation;
532         else
533                 ups->brush_rotation_sec = 0.0f;
534 }
535
536 void paint_calculate_rake_rotation(UnifiedPaintSettings *ups, Brush *brush, const float mouse_pos[2])
537 {
538         if ((brush->mtex.brush_angle_mode & MTEX_ANGLE_RAKE) || (brush->mask_mtex.brush_angle_mode & MTEX_ANGLE_RAKE)) {
539                 const float u = 0.5f;
540                 const float r = RAKE_THRESHHOLD;
541                 float rotation;
542
543                 float dpos[2];
544                 sub_v2_v2v2(dpos, ups->last_rake, mouse_pos);
545
546                 if (len_squared_v2(dpos) >= r * r) {
547                         rotation = atan2f(dpos[0], dpos[1]);
548
549                         interp_v2_v2v2(ups->last_rake, ups->last_rake,
550                                        mouse_pos, u);
551
552                         ups->last_rake_angle = rotation;
553
554                         update_brush_rake_rotation(ups, brush, rotation);
555                 }
556                 /* make sure we reset here to the last rotation to avoid accumulating
557                  * values in case a random rotation is also added */
558                 else {
559                         update_brush_rake_rotation(ups, brush, ups->last_rake_angle);
560                 }
561         }
562         else {
563                 ups->brush_rotation = ups->brush_rotation_sec = 0.0f;
564         }
565 }
566
567 void BKE_sculptsession_free_deformMats(SculptSession *ss)
568 {
569         MEM_SAFE_FREE(ss->orig_cos);
570         MEM_SAFE_FREE(ss->deform_cos);
571         MEM_SAFE_FREE(ss->deform_imats);
572 }
573
574 /* Write out the sculpt dynamic-topology BMesh to the Mesh */
575 static void sculptsession_bm_to_me_update_data_only(Object *ob, bool reorder)
576 {
577         SculptSession *ss = ob->sculpt;
578
579         if (ss->bm) {
580                 if (ob->data) {
581                         BMIter iter;
582                         BMFace *efa;
583                         BM_ITER_MESH (efa, &iter, ss->bm, BM_FACES_OF_MESH) {
584                                 BM_elem_flag_set(efa, BM_ELEM_SMOOTH, ss->bm_smooth_shading);
585                         }
586                         if (reorder)
587                                 BM_log_mesh_elems_reorder(ss->bm, ss->bm_log);
588                         BM_mesh_bm_to_me(ss->bm, ob->data, false);
589                 }
590         }
591 }
592
593 void BKE_sculptsession_bm_to_me(Object *ob, bool reorder)
594 {
595         if (ob && ob->sculpt) {
596                 sculptsession_bm_to_me_update_data_only(ob, reorder);
597
598                 /* ensure the objects DerivedMesh mesh doesn't hold onto arrays now realloc'd in the mesh [#34473] */
599                 DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
600         }
601 }
602
603 void BKE_sculptsession_bm_to_me_for_render(Object *object)
604 {
605         if (object && object->sculpt) {
606                 if (object->sculpt->bm) {
607                         /* Ensure no points to old arrays are stored in DM
608                          *
609                          * Apparently, we could not use DAG_id_tag_update
610                          * here because this will lead to the while object
611                          * surface to disappear, so we'll release DM in place.
612                          */
613                         BKE_object_free_derived_caches(object);
614
615                         if (object->sculpt->pbvh) {
616                                 BKE_pbvh_free(object->sculpt->pbvh);
617                                 object->sculpt->pbvh = NULL;
618                         }
619
620                         sculptsession_bm_to_me_update_data_only(object, false);
621
622                         /* In contrast with sculptsession_bm_to_me no need in
623                          * DAG tag update here - derived mesh was freed and
624                          * old pointers are nowhere stored.
625                          */
626                 }
627         }
628 }
629
630 void BKE_sculptsession_free(Object *ob)
631 {
632         if (ob && ob->sculpt) {
633                 SculptSession *ss = ob->sculpt;
634                 DerivedMesh *dm = ob->derivedFinal;
635
636                 if (ss->bm) {
637                         BKE_sculptsession_bm_to_me(ob, true);
638                         BM_mesh_free(ss->bm);
639                 }
640
641                 if (ss->pbvh)
642                         BKE_pbvh_free(ss->pbvh);
643                 if (ss->bm_log)
644                         BM_log_free(ss->bm_log);
645
646                 if (dm && dm->getPBVH)
647                         dm->getPBVH(NULL, dm);  /* signal to clear */
648
649                 if (ss->texcache)
650                         MEM_freeN(ss->texcache);
651
652                 if (ss->tex_pool)
653                         BKE_image_pool_free(ss->tex_pool);
654
655                 if (ss->layer_co)
656                         MEM_freeN(ss->layer_co);
657
658                 if (ss->orig_cos)
659                         MEM_freeN(ss->orig_cos);
660                 if (ss->deform_cos)
661                         MEM_freeN(ss->deform_cos);
662                 if (ss->deform_imats)
663                         MEM_freeN(ss->deform_imats);
664
665                 MEM_freeN(ss);
666
667                 ob->sculpt = NULL;
668         }
669 }
670
671 /* Sculpt mode handles multires differently from regular meshes, but only if
672  * it's the last modifier on the stack and it is not on the first level */
673 MultiresModifierData *BKE_sculpt_multires_active(Scene *scene, Object *ob)
674 {
675         Mesh *me = (Mesh *)ob->data;
676         ModifierData *md;
677         VirtualModifierData virtualModifierData;
678
679         if (ob->sculpt && ob->sculpt->bm) {
680                 /* can't combine multires and dynamic topology */
681                 return NULL;
682         }
683
684         if (!CustomData_get_layer(&me->ldata, CD_MDISPS)) {
685                 /* multires can't work without displacement layer */
686                 return NULL;
687         }
688
689         for (md = modifiers_getVirtualModifierList(ob, &virtualModifierData); md; md = md->next) {
690                 if (md->type == eModifierType_Multires) {
691                         MultiresModifierData *mmd = (MultiresModifierData *)md;
692
693                         if (!modifier_isEnabled(scene, md, eModifierMode_Realtime))
694                                 continue;
695
696                         if (mmd->sculptlvl > 0) return mmd;
697                         else return NULL;
698                 }
699         }
700
701         return NULL;
702 }
703
704
705 /* Checks if there are any supported deformation modifiers active */
706 static bool sculpt_modifiers_active(Scene *scene, Sculpt *sd, Object *ob)
707 {
708         ModifierData *md;
709         Mesh *me = (Mesh *)ob->data;
710         MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
711         VirtualModifierData virtualModifierData;
712
713         if (mmd || ob->sculpt->bm)
714                 return false;
715
716         /* non-locked shape keys could be handled in the same way as deformed mesh */
717         if ((ob->shapeflag & OB_SHAPE_LOCK) == 0 && me->key && ob->shapenr)
718                 return true;
719
720         md = modifiers_getVirtualModifierList(ob, &virtualModifierData);
721
722         /* exception for shape keys because we can edit those */
723         for (; md; md = md->next) {
724                 const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
725                 if (!modifier_isEnabled(scene, md, eModifierMode_Realtime)) continue;
726                 if (ELEM(md->type, eModifierType_ShapeKey, eModifierType_Multires)) continue;
727
728                 if (mti->type == eModifierTypeType_OnlyDeform) return true;
729                 else if ((sd->flags & SCULPT_ONLY_DEFORM) == 0) return true;
730         }
731
732         return false;
733 }
734
735 /**
736  * \param need_mask So the DerivedMesh thats returned has mask data
737  */
738 void BKE_sculpt_update_mesh_elements(Scene *scene, Sculpt *sd, Object *ob,
739                                      bool need_pmap, bool need_mask)
740 {
741         DerivedMesh *dm;
742         SculptSession *ss = ob->sculpt;
743         Mesh *me = ob->data;
744         MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
745
746         ss->modifiers_active = sculpt_modifiers_active(scene, sd, ob);
747         ss->show_diffuse_color = (sd->flags & SCULPT_SHOW_DIFFUSE) != 0;
748
749         if (need_mask) {
750                 if (mmd == NULL) {
751                         if (!CustomData_has_layer(&me->vdata, CD_PAINT_MASK)) {
752                                 BKE_sculpt_mask_layers_ensure(ob, NULL);
753                         }
754                 }
755                 else {
756                         if (!CustomData_has_layer(&me->ldata, CD_GRID_PAINT_MASK)) {
757 #if 1
758                                 BKE_sculpt_mask_layers_ensure(ob, mmd);
759 #else                           /* if we wanted to support adding mask data while multi-res painting, we would need to do this */
760                                 if ((ED_sculpt_mask_layers_ensure(ob, mmd) & ED_SCULPT_MASK_LAYER_CALC_LOOP)) {
761                                         /* remake the derived mesh */
762                                         ob->recalc |= OB_RECALC_DATA;
763                                         BKE_object_handle_update(scene, ob);
764                                 }
765 #endif
766                         }
767                 }
768         }
769
770         /* BMESH ONLY --- at some point we should move sculpt code to use polygons only - but for now it needs tessfaces */
771         if (ss->bm) {
772                 BKE_mesh_tessface_clear(me);
773         }
774         else {
775                 BKE_mesh_tessface_ensure(me);
776         }
777
778         if (!mmd) ss->kb = BKE_keyblock_from_object(ob);
779         else ss->kb = NULL;
780
781         /* needs to be called after we ensure tessface */
782         dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
783
784         if (mmd) {
785                 ss->multires = mmd;
786                 ss->totvert = dm->getNumVerts(dm);
787                 ss->totpoly = dm->getNumPolys(dm);
788                 ss->mvert = NULL;
789                 ss->mpoly = NULL;
790                 ss->mloop = NULL;
791                 ss->face_normals = NULL;
792         }
793         else {
794                 ss->totvert = me->totvert;
795                 ss->totpoly = me->totpoly;
796                 ss->mvert = me->mvert;
797                 ss->mpoly = me->mpoly;
798                 ss->mloop = me->mloop;
799                 ss->face_normals = NULL;
800                 ss->multires = NULL;
801                 ss->vmask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
802         }
803
804         ss->pbvh = dm->getPBVH(ob, dm);
805         ss->pmap = (need_pmap && dm->getPolyMap) ? dm->getPolyMap(ob, dm) : NULL;
806
807         pbvh_show_diffuse_color_set(ss->pbvh, ss->show_diffuse_color);
808
809         if (ss->modifiers_active) {
810                 if (!ss->orig_cos) {
811                         int a;
812
813                         BKE_sculptsession_free_deformMats(ss);
814
815                         ss->orig_cos = (ss->kb) ? BKE_keyblock_convert_to_vertcos(ob, ss->kb) : BKE_mesh_vertexCos_get(me, NULL);
816
817                         BKE_crazyspace_build_sculpt(scene, ob, &ss->deform_imats, &ss->deform_cos);
818                         BKE_pbvh_apply_vertCos(ss->pbvh, ss->deform_cos);
819
820                         for (a = 0; a < me->totvert; ++a) {
821                                 invert_m3(ss->deform_imats[a]);
822                         }
823                 }
824         }
825         else {
826                 BKE_sculptsession_free_deformMats(ss);
827         }
828
829         if (ss->kb != NULL && ss->deform_cos == NULL) {
830                 ss->deform_cos = BKE_keyblock_convert_to_vertcos(ob, ss->kb);
831         }
832
833         /* if pbvh is deformed, key block is already applied to it */
834         if (ss->kb) {
835                 bool pbvh_deformd = BKE_pbvh_isDeformed(ss->pbvh);
836                 if (!pbvh_deformd || ss->deform_cos == NULL) {
837                         float (*vertCos)[3] = BKE_keyblock_convert_to_vertcos(ob, ss->kb);
838
839                         if (vertCos) {
840                                 if (!pbvh_deformd) {
841                                         /* apply shape keys coordinates to PBVH */
842                                         BKE_pbvh_apply_vertCos(ss->pbvh, vertCos);
843                                 }
844                                 if (ss->deform_cos == NULL) {
845                                         ss->deform_cos = vertCos;
846                                 }
847                                 if (vertCos != ss->deform_cos) {
848                                         MEM_freeN(vertCos);
849                                 }
850                         }
851                 }
852         }
853 }
854
855 int BKE_sculpt_mask_layers_ensure(Object *ob, MultiresModifierData *mmd)
856 {
857         const float *paint_mask;
858         Mesh *me = ob->data;
859         int ret = 0;
860
861         paint_mask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
862
863         /* if multires is active, create a grid paint mask layer if there
864          * isn't one already */
865         if (mmd && !CustomData_has_layer(&me->ldata, CD_GRID_PAINT_MASK)) {
866                 GridPaintMask *gmask;
867                 int level = max_ii(1, mmd->sculptlvl);
868                 int gridsize = BKE_ccg_gridsize(level);
869                 int gridarea = gridsize * gridsize;
870                 int i, j;
871
872                 gmask = CustomData_add_layer(&me->ldata, CD_GRID_PAINT_MASK,
873                                              CD_CALLOC, NULL, me->totloop);
874
875                 for (i = 0; i < me->totloop; i++) {
876                         GridPaintMask *gpm = &gmask[i];
877
878                         gpm->level = level;
879                         gpm->data = MEM_callocN(sizeof(float) * gridarea,
880                                                 "GridPaintMask.data");
881                 }
882
883                 /* if vertices already have mask, copy into multires data */
884                 if (paint_mask) {
885                         for (i = 0; i < me->totpoly; i++) {
886                                 const MPoly *p = &me->mpoly[i];
887                                 float avg = 0;
888
889                                 /* mask center */
890                                 for (j = 0; j < p->totloop; j++) {
891                                         const MLoop *l = &me->mloop[p->loopstart + j];
892                                         avg += paint_mask[l->v];
893                                 }
894                                 avg /= (float)p->totloop;
895
896                                 /* fill in multires mask corner */
897                                 for (j = 0; j < p->totloop; j++) {
898                                         GridPaintMask *gpm = &gmask[p->loopstart + j];
899                                         const MLoop *l = &me->mloop[p->loopstart + j];
900                                         const MLoop *prev = ME_POLY_LOOP_PREV(me->mloop, p, j);
901                                         const MLoop *next = ME_POLY_LOOP_NEXT(me->mloop, p, j);
902
903                                         gpm->data[0] = avg;
904                                         gpm->data[1] = (paint_mask[l->v] +
905                                                         paint_mask[next->v]) * 0.5f;
906                                         gpm->data[2] = (paint_mask[l->v] +
907                                                         paint_mask[prev->v]) * 0.5f;
908                                         gpm->data[3] = paint_mask[l->v];
909                                 }
910                         }
911                 }
912
913                 ret |= SCULPT_MASK_LAYER_CALC_LOOP;
914         }
915
916         /* create vertex paint mask layer if there isn't one already */
917         if (!paint_mask) {
918                 CustomData_add_layer(&me->vdata, CD_PAINT_MASK,
919                                      CD_CALLOC, NULL, me->totvert);
920                 ret |= SCULPT_MASK_LAYER_CALC_VERT;
921         }
922
923         return ret;
924 }