Bevel mod: cleanup flags and extra data.
[blender.git] / source / blender / editors / sculpt_paint / paint_hide.c
1 /*
2  * This program is free software; you can redistribute it and/or
3  * modify it under the terms of the GNU General Public License
4  * as published by the Free Software Foundation; either version 2
5  * of the License, or (at your option) any later version.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * along with this program; if not, write to the Free Software  Foundation,
14  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15  *
16  * The Original Code is Copyright (C) 2010 by Nicholas Bishop
17  * All rights reserved.
18  * Implements the PBVH node hiding operator
19  */
20
21 /** \file
22  * \ingroup edsculpt
23  */
24
25 #include "MEM_guardedalloc.h"
26
27 #include "BLI_bitmap.h"
28 #include "BLI_math_vector.h"
29 #include "BLI_utildefines.h"
30
31 #include "DNA_mesh_types.h"
32 #include "DNA_meshdata_types.h"
33 #include "DNA_object_types.h"
34 #include "DNA_scene_types.h"
35
36 #include "BKE_pbvh.h"
37 #include "BKE_ccg.h"
38 #include "BKE_context.h"
39 #include "BKE_mesh.h"
40 #include "BKE_mesh_runtime.h"
41 #include "BKE_multires.h"
42 #include "BKE_paint.h"
43 #include "BKE_subsurf.h"
44
45 #include "DEG_depsgraph.h"
46
47 #include "WM_api.h"
48 #include "WM_types.h"
49
50 #include "ED_screen.h"
51 #include "ED_view3d.h"
52
53 #include "RNA_access.h"
54 #include "RNA_define.h"
55
56 #include "bmesh.h"
57
58 #include "paint_intern.h"
59 #include "sculpt_intern.h" /* for undo push */
60
61 #include <assert.h>
62
63 /* return true if the element should be hidden/shown */
64 static bool is_effected(
65         PartialVisArea area,
66         float planes[4][4],
67         const float co[3],
68         const float mask)
69 {
70         if (area == PARTIALVIS_ALL)
71                 return 1;
72         else if (area == PARTIALVIS_MASKED) {
73                 return mask > 0.5f;
74         }
75         else {
76                 bool inside = isect_point_planes_v3(planes, 4, co);
77                 return ((inside && area == PARTIALVIS_INSIDE) ||
78                         (!inside && area == PARTIALVIS_OUTSIDE));
79         }
80 }
81
82 static void partialvis_update_mesh(
83         Object *ob,
84         PBVH *pbvh,
85         PBVHNode *node,
86         PartialVisAction action,
87         PartialVisArea area,
88         float planes[4][4])
89 {
90         Mesh *me = ob->data;
91         MVert *mvert;
92         const float *paint_mask;
93         const int *vert_indices;
94         int totvert, i;
95         bool any_changed = false, any_visible = false;
96
97         BKE_pbvh_node_num_verts(pbvh, node, NULL, &totvert);
98         BKE_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
99         paint_mask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
100
101         sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
102
103         for (i = 0; i < totvert; i++) {
104                 MVert *v = &mvert[vert_indices[i]];
105                 float vmask = paint_mask ? paint_mask[vert_indices[i]] : 0;
106
107                 /* hide vertex if in the hide volume */
108                 if (is_effected(area, planes, v->co, vmask)) {
109                         if (action == PARTIALVIS_HIDE)
110                                 v->flag |= ME_HIDE;
111                         else
112                                 v->flag &= ~ME_HIDE;
113                         any_changed = true;
114                 }
115
116                 if (!(v->flag & ME_HIDE))
117                         any_visible = true;
118         }
119
120         if (any_changed) {
121                 BKE_pbvh_node_mark_rebuild_draw(node);
122                 BKE_pbvh_node_fully_hidden_set(node, !any_visible);
123         }
124 }
125
126 /* Hide or show elements in multires grids with a special GridFlags
127  * customdata layer. */
128 static void partialvis_update_grids(
129         Object *ob,
130         PBVH *pbvh,
131         PBVHNode *node,
132         PartialVisAction action,
133         PartialVisArea area,
134         float planes[4][4])
135 {
136         CCGElem **grids;
137         CCGKey key;
138         BLI_bitmap **grid_hidden;
139         int *grid_indices, totgrid, i;
140         bool any_changed = false, any_visible = false;
141
142
143         /* get PBVH data */
144         BKE_pbvh_node_get_grids(
145                 pbvh, node,
146                 &grid_indices, &totgrid, NULL, NULL,
147                 &grids);
148         grid_hidden = BKE_pbvh_grid_hidden(pbvh);
149         BKE_pbvh_get_grid_key(pbvh, &key);
150
151         sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
152
153         for (i = 0; i < totgrid; i++) {
154                 int any_hidden = 0;
155                 int g = grid_indices[i], x, y;
156                 BLI_bitmap *gh = grid_hidden[g];
157
158                 if (!gh) {
159                         switch (action) {
160                                 case PARTIALVIS_HIDE:
161                                         /* create grid flags data */
162                                         gh = grid_hidden[g] = BLI_BITMAP_NEW(
163                                                 key.grid_area,
164                                                 "partialvis_update_grids");
165                                         break;
166                                 case PARTIALVIS_SHOW:
167                                         /* entire grid is visible, nothing to show */
168                                         continue;
169                         }
170                 }
171                 else if (action == PARTIALVIS_SHOW && area == PARTIALVIS_ALL) {
172                         /* special case if we're showing all, just free the
173                          * grid */
174                         MEM_freeN(gh);
175                         grid_hidden[g] = NULL;
176                         any_changed = true;
177                         any_visible = true;
178                         continue;
179                 }
180
181                 for (y = 0; y < key.grid_size; y++) {
182                         for (x = 0; x < key.grid_size; x++) {
183                                 CCGElem *elem = CCG_grid_elem(&key, grids[g], x, y);
184                                 const float *co = CCG_elem_co(&key, elem);
185                                 float mask = key.has_mask ? *CCG_elem_mask(&key, elem) : 0.0f;
186
187                                 /* skip grid element if not in the effected area */
188                                 if (is_effected(area, planes, co, mask)) {
189                                         /* set or clear the hide flag */
190                                         BLI_BITMAP_SET(
191                                                 gh, y * key.grid_size + x,
192                                                 action == PARTIALVIS_HIDE);
193
194                                         any_changed = true;
195                                 }
196
197                                 /* keep track of whether any elements are still hidden */
198                                 if (BLI_BITMAP_TEST(gh, y * key.grid_size + x))
199                                         any_hidden = true;
200                                 else
201                                         any_visible = true;
202                         }
203                 }
204
205                 /* if everything in the grid is now visible, free the grid
206                  * flags */
207                 if (!any_hidden) {
208                         MEM_freeN(gh);
209                         grid_hidden[g] = NULL;
210                 }
211         }
212
213         /* mark updates if anything was hidden/shown */
214         if (any_changed) {
215                 BKE_pbvh_node_mark_rebuild_draw(node);
216                 BKE_pbvh_node_fully_hidden_set(node, !any_visible);
217                 multires_mark_as_modified(ob, MULTIRES_HIDDEN_MODIFIED);
218         }
219 }
220
221 static void partialvis_update_bmesh_verts(
222         BMesh *bm,
223         GSet *verts,
224         PartialVisAction action,
225         PartialVisArea area,
226         float planes[4][4],
227         bool *any_changed,
228         bool *any_visible)
229 {
230         GSetIterator gs_iter;
231
232         GSET_ITER (gs_iter, verts) {
233                 BMVert *v = BLI_gsetIterator_getKey(&gs_iter);
234                 float *vmask = CustomData_bmesh_get(
235                         &bm->vdata, v->head.data, CD_PAINT_MASK);
236
237                 /* hide vertex if in the hide volume */
238                 if (is_effected(area, planes, v->co, *vmask)) {
239                         if (action == PARTIALVIS_HIDE)
240                                 BM_elem_flag_enable(v, BM_ELEM_HIDDEN);
241                         else
242                                 BM_elem_flag_disable(v, BM_ELEM_HIDDEN);
243                         (*any_changed) = true;
244                 }
245
246                 if (!BM_elem_flag_test(v, BM_ELEM_HIDDEN))
247                         (*any_visible) = true;
248         }
249 }
250
251 static void partialvis_update_bmesh_faces(GSet *faces)
252 {
253         GSetIterator gs_iter;
254
255         GSET_ITER (gs_iter, faces) {
256                 BMFace *f = BLI_gsetIterator_getKey(&gs_iter);
257
258                 if (paint_is_bmesh_face_hidden(f))
259                         BM_elem_flag_enable(f, BM_ELEM_HIDDEN);
260                 else
261                         BM_elem_flag_disable(f, BM_ELEM_HIDDEN);
262         }
263 }
264
265 static void partialvis_update_bmesh(
266         Object *ob,
267         PBVH *pbvh,
268         PBVHNode *node,
269         PartialVisAction action,
270         PartialVisArea area,
271         float planes[4][4])
272 {
273         BMesh *bm;
274         GSet *unique, *other, *faces;
275         bool any_changed = false, any_visible = false;
276
277         bm = BKE_pbvh_get_bmesh(pbvh);
278         unique = BKE_pbvh_bmesh_node_unique_verts(node);
279         other = BKE_pbvh_bmesh_node_other_verts(node);
280         faces = BKE_pbvh_bmesh_node_faces(node);
281
282         sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
283
284         partialvis_update_bmesh_verts(
285                 bm,
286                 unique,
287                 action,
288                 area,
289                 planes,
290                 &any_changed,
291                 &any_visible);
292
293         partialvis_update_bmesh_verts(
294                 bm,
295                 other,
296                 action,
297                 area,
298                 planes,
299                 &any_changed,
300                 &any_visible);
301
302         /* finally loop over node faces and tag the ones that are fully hidden */
303         partialvis_update_bmesh_faces(faces);
304
305         if (any_changed) {
306                 BKE_pbvh_node_mark_rebuild_draw(node);
307                 BKE_pbvh_node_fully_hidden_set(node, !any_visible);
308         }
309 }
310
311 static void rect_from_props(rcti *rect, PointerRNA *ptr)
312 {
313         rect->xmin = RNA_int_get(ptr, "xmin");
314         rect->ymin = RNA_int_get(ptr, "ymin");
315         rect->xmax = RNA_int_get(ptr, "xmax");
316         rect->ymax = RNA_int_get(ptr, "ymax");
317 }
318
319 static void clip_planes_from_rect(
320         bContext *C,
321         float clip_planes[4][4],
322         const rcti *rect)
323 {
324         ViewContext vc;
325         BoundBox bb;
326
327         view3d_operator_needs_opengl(C);
328         ED_view3d_viewcontext_init(C, &vc);
329         ED_view3d_clipping_calc(&bb, clip_planes, vc.ar, vc.obact, rect);
330         negate_m4(clip_planes);
331 }
332
333 /* If mode is inside, get all PBVH nodes that lie at least partially
334  * inside the clip_planes volume. If mode is outside, get all nodes
335  * that lie at least partially outside the volume. If showing all, get
336  * all nodes. */
337 static void get_pbvh_nodes(
338         PBVH *pbvh,
339         PBVHNode ***nodes,
340         int *totnode,
341         float clip_planes[4][4],
342         PartialVisArea mode)
343 {
344         BKE_pbvh_SearchCallback cb = NULL;
345
346         /* select search callback */
347         switch (mode) {
348                 case PARTIALVIS_INSIDE:
349                         cb = BKE_pbvh_node_planes_contain_AABB;
350                         break;
351                 case PARTIALVIS_OUTSIDE:
352                         cb = BKE_pbvh_node_planes_exclude_AABB;
353                         break;
354                 case PARTIALVIS_ALL:
355                 case PARTIALVIS_MASKED:
356                         break;
357         }
358
359         BKE_pbvh_search_gather(pbvh, cb, clip_planes, nodes, totnode);
360 }
361
362 static int hide_show_exec(bContext *C, wmOperator *op)
363 {
364         ARegion *ar = CTX_wm_region(C);
365         Object *ob = CTX_data_active_object(C);
366         Depsgraph *depsgraph = CTX_data_depsgraph(C);
367         Mesh *me = ob->data;
368         PartialVisAction action;
369         PartialVisArea area;
370         PBVH *pbvh;
371         PBVHNode **nodes;
372         PBVHType pbvh_type;
373         float clip_planes[4][4];
374         rcti rect;
375         int totnode, i;
376
377         /* read operator properties */
378         action = RNA_enum_get(op->ptr, "action");
379         area = RNA_enum_get(op->ptr, "area");
380         rect_from_props(&rect, op->ptr);
381
382         clip_planes_from_rect(C, clip_planes, &rect);
383
384         pbvh = BKE_sculpt_object_pbvh_ensure(depsgraph, ob);
385         BLI_assert(ob->sculpt->pbvh == pbvh);
386
387         get_pbvh_nodes(pbvh, &nodes, &totnode, clip_planes, area);
388         pbvh_type = BKE_pbvh_type(pbvh);
389
390         /* start undo */
391         switch (action) {
392                 case PARTIALVIS_HIDE:
393                         sculpt_undo_push_begin("Hide area");
394                         break;
395                 case PARTIALVIS_SHOW:
396                         sculpt_undo_push_begin("Show area");
397                         break;
398         }
399
400         for (i = 0; i < totnode; i++) {
401                 switch (pbvh_type) {
402                         case PBVH_FACES:
403                                 partialvis_update_mesh(ob, pbvh, nodes[i], action, area, clip_planes);
404                                 break;
405                         case PBVH_GRIDS:
406                                 partialvis_update_grids(ob, pbvh, nodes[i], action, area, clip_planes);
407                                 break;
408                         case PBVH_BMESH:
409                                 partialvis_update_bmesh(ob, pbvh, nodes[i], action, area, clip_planes);
410                                 break;
411                 }
412         }
413
414         if (nodes)
415                 MEM_freeN(nodes);
416
417         /* end undo */
418         sculpt_undo_push_end();
419
420         /* ensure that edges and faces get hidden as well (not used by
421          * sculpt but it looks wrong when entering editmode otherwise) */
422         if (pbvh_type == PBVH_FACES) {
423                 BKE_mesh_flush_hidden_from_verts(me);
424         }
425
426         ED_region_tag_redraw(ar);
427
428         return OPERATOR_FINISHED;
429 }
430
431 static int hide_show_invoke(bContext *C, wmOperator *op, const wmEvent *event)
432 {
433         PartialVisArea area = RNA_enum_get(op->ptr, "area");
434
435         if (!ELEM(area, PARTIALVIS_ALL, PARTIALVIS_MASKED))
436                 return WM_gesture_box_invoke(C, op, event);
437         else
438                 return op->type->exec(C, op);
439 }
440
441 void PAINT_OT_hide_show(struct wmOperatorType *ot)
442 {
443         static const EnumPropertyItem action_items[] = {
444                 {PARTIALVIS_HIDE, "HIDE", 0, "Hide", "Hide vertices"},
445                 {PARTIALVIS_SHOW, "SHOW", 0, "Show", "Show vertices"},
446                 {0, NULL, 0, NULL, NULL},
447         };
448
449         static const EnumPropertyItem area_items[] = {
450                 {PARTIALVIS_OUTSIDE, "OUTSIDE", 0, "Outside", "Hide or show vertices outside the selection"},
451                 {PARTIALVIS_INSIDE, "INSIDE", 0, "Inside", "Hide or show vertices inside the selection"},
452                 {PARTIALVIS_ALL, "ALL", 0, "All", "Hide or show all vertices"},
453                 {PARTIALVIS_MASKED, "MASKED", 0, "Masked", "Hide or show vertices that are masked (minimum mask value of 0.5)"},
454                 {0, NULL, 0, NULL, NULL},
455         };
456
457         /* identifiers */
458         ot->name = "Hide/Show";
459         ot->idname = "PAINT_OT_hide_show";
460         ot->description = "Hide/show some vertices";
461
462         /* api callbacks */
463         ot->invoke = hide_show_invoke;
464         ot->modal = WM_gesture_box_modal;
465         ot->exec = hide_show_exec;
466         /* sculpt-only for now */
467         ot->poll = sculpt_mode_poll_view3d;
468
469         ot->flag = OPTYPE_REGISTER;
470
471         /* rna */
472         RNA_def_enum(ot->srna, "action", action_items, PARTIALVIS_HIDE,
473                      "Action", "Whether to hide or show vertices");
474         RNA_def_enum(ot->srna, "area", area_items, PARTIALVIS_INSIDE,
475                      "Area", "Which vertices to hide or show");
476
477         WM_operator_properties_border(ot);
478 }