Quite some warnings...
[blender.git] / source / blender / editors / sculpt_paint / paint_hide.c
1 /*
2  * ***** BEGIN GPL LICENSE BLOCK *****
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version 2
7  * of the License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software  Foundation,
16  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17  *
18  * The Original Code is Copyright (C) 2010 by Nicholas Bishop
19  * All rights reserved.
20  *
21  * The Original Code is: all of this file.
22  *
23  * Contributor(s):
24  *
25  * ***** END GPL LICENSE BLOCK *****
26  *
27  * Implements the PBVH node hiding operator
28  *
29  */
30
31 /** \file blender/editors/sculpt_paint/paint_hide.c
32  *  \ingroup edsculpt
33  */
34
35 #include "MEM_guardedalloc.h"
36
37 #include "BLI_bitmap.h"
38 #include "BLI_listbase.h"
39 #include "BLI_math_vector.h"
40 #include "BLI_pbvh.h"
41 #include "BLI_utildefines.h"
42
43 #include "DNA_mesh_types.h"
44 #include "DNA_meshdata_types.h"
45 #include "DNA_object_types.h"
46 #include "DNA_scene_types.h"
47
48 #include "BKE_ccg.h"
49 #include "BKE_context.h"
50 #include "BKE_DerivedMesh.h"
51 #include "BKE_mesh.h"
52 #include "BKE_multires.h"
53 #include "BKE_paint.h"
54 #include "BKE_subsurf.h"
55
56 #include "BIF_glutil.h"
57
58 #include "WM_api.h"
59 #include "WM_types.h"
60
61 #include "ED_screen.h"
62 #include "ED_view3d.h"
63
64 #include "RNA_access.h"
65 #include "RNA_define.h"
66
67 #include "paint_intern.h"
68 #include "sculpt_intern.h" /* for undo push */
69
70 #include <assert.h>
71
72 static int planes_contain_v3(float (*planes)[4], int totplane, const float p[3])
73 {
74         int i;
75
76         for (i = 0; i < totplane; i++) {
77                 if (dot_v3v3(planes[i], p) + planes[i][3] > 0)
78                         return 0;
79         }
80
81         return 1;
82 }
83
84 /* return true if the element should be hidden/shown */
85 static int is_effected(PartialVisArea area,
86                        float planes[4][4],
87                        const float co[3],
88                        const float mask)
89 {
90         if (area == PARTIALVIS_ALL)
91                 return 1;
92         else if (area == PARTIALVIS_MASKED) {
93                 return mask > 0.5f;
94         }
95         else {
96                 int inside = planes_contain_v3(planes, 4, co);
97                 return ((inside && area == PARTIALVIS_INSIDE) ||
98                         (!inside && area == PARTIALVIS_OUTSIDE));
99         }
100 }
101
102 static void partialvis_update_mesh(Object *ob,
103                                    PBVH *pbvh,
104                                    PBVHNode *node,
105                                    PartialVisAction action,
106                                    PartialVisArea area,
107                                    float planes[4][4])
108 {
109         Mesh *me = ob->data;
110         MVert *mvert;
111         float *paint_mask;
112         int *vert_indices;
113         int any_changed = 0, any_visible = 0, totvert, i;
114                         
115         BLI_pbvh_node_num_verts(pbvh, node, NULL, &totvert);
116         BLI_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
117         paint_mask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
118
119         sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
120
121         for (i = 0; i < totvert; i++) {
122                 MVert *v = &mvert[vert_indices[i]];
123                 float vmask = paint_mask ? paint_mask[vert_indices[i]] : 0;
124
125                 /* hide vertex if in the hide volume */
126                 if (is_effected(area, planes, v->co, vmask)) {
127                         if (action == PARTIALVIS_HIDE)
128                                 v->flag |= ME_HIDE;
129                         else
130                                 v->flag &= ~ME_HIDE;
131                         any_changed = 1;
132                 }
133
134                 if (!(v->flag & ME_HIDE))
135                         any_visible = 1;
136         }
137
138         if (any_changed) {
139                 BLI_pbvh_node_mark_rebuild_draw(node);
140                 BLI_pbvh_node_fully_hidden_set(node, !any_visible);
141         }
142 }
143
144 /* Hide or show elements in multires grids with a special GridFlags
145  * customdata layer. */
146 static void partialvis_update_grids(Object *ob,
147                                     PBVH *pbvh,
148                                     PBVHNode *node,
149                                     PartialVisAction action,
150                                     PartialVisArea area,
151                                     float planes[4][4])
152 {
153         CCGElem **grids;
154         CCGKey key;
155         BLI_bitmap *grid_hidden;
156         int any_visible = 0;
157         int *grid_indices, totgrid, any_changed, i;
158
159         /* get PBVH data */
160         BLI_pbvh_node_get_grids(pbvh, node,
161                                 &grid_indices, &totgrid, NULL, NULL,
162                                 &grids, NULL);
163         grid_hidden = BLI_pbvh_grid_hidden(pbvh);
164         BLI_pbvh_get_grid_key(pbvh, &key);
165         
166         sculpt_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
167         
168         any_changed = 0;
169         for (i = 0; i < totgrid; i++) {
170                 int any_hidden = 0;
171                 int g = grid_indices[i], x, y;
172                 BLI_bitmap gh = grid_hidden[g];
173
174                 if (!gh) {
175                         switch (action) {
176                                 case PARTIALVIS_HIDE:
177                                         /* create grid flags data */
178                                         gh = grid_hidden[g] = BLI_BITMAP_NEW(key.grid_area,
179                                                                              "partialvis_update_grids");
180                                         break;
181                                 case PARTIALVIS_SHOW:
182                                         /* entire grid is visible, nothing to show */
183                                         continue;
184                         }
185                 }
186                 else if (action == PARTIALVIS_SHOW && area == PARTIALVIS_ALL) {
187                         /* special case if we're showing all, just free the
188                          * grid */
189                         MEM_freeN(gh);
190                         grid_hidden[g] = NULL;
191                         any_changed = 1;
192                         any_visible = 1;
193                         continue;
194                 }
195
196                 for (y = 0; y < key.grid_size; y++) {
197                         for (x = 0; x < key.grid_size; x++) {
198                                 CCGElem *elem = CCG_grid_elem(&key, grids[g], x, y);
199                                 const float *co = CCG_elem_co(&key, elem);
200                                 float mask = *CCG_elem_mask(&key, elem);
201
202                                 /* skip grid element if not in the effected area */
203                                 if (is_effected(area, planes, co, mask)) {
204                                         /* set or clear the hide flag */
205                                         BLI_BITMAP_MODIFY(gh, y * key.grid_size + x,
206                                                           action == PARTIALVIS_HIDE);
207
208                                         any_changed = 1;
209                                 }
210
211                                 /* keep track of whether any elements are still hidden */
212                                 if (BLI_BITMAP_GET(gh, y * key.grid_size + x))
213                                         any_hidden = 1;
214                                 else
215                                         any_visible = 1;
216                         }
217                 }
218
219                 /* if everything in the grid is now visible, free the grid
220                  * flags */
221                 if (!any_hidden) {
222                         MEM_freeN(gh);
223                         grid_hidden[g] = NULL;
224                 }
225         }
226
227         /* mark updates if anything was hidden/shown */
228         if (any_changed) {
229                 BLI_pbvh_node_mark_rebuild_draw(node);
230                 BLI_pbvh_node_fully_hidden_set(node, !any_visible);
231                 multires_mark_as_modified(ob, MULTIRES_HIDDEN_MODIFIED);
232         }
233 }
234
235 static void rect_from_props(rcti *rect, PointerRNA *ptr)
236 {
237         rect->xmin = RNA_int_get(ptr, "xmin");
238         rect->ymin = RNA_int_get(ptr, "ymin");
239         rect->xmax = RNA_int_get(ptr, "xmax");
240         rect->ymax = RNA_int_get(ptr, "ymax");
241 }
242
243 static void clip_planes_from_rect(bContext *C,
244                                   float clip_planes[4][4],
245                                   const rcti *rect)
246 {
247         ViewContext vc;
248         BoundBox bb;
249         bglMats mats = {{0}};
250         
251         view3d_operator_needs_opengl(C);
252         view3d_set_viewcontext(C, &vc);
253         view3d_get_transformation(vc.ar, vc.rv3d, vc.obact, &mats);
254         ED_view3d_calc_clipping(&bb, clip_planes, &mats, rect);
255         mul_m4_fl(clip_planes, -1.0f);
256 }
257
258 /* If mode is inside, get all PBVH nodes that lie at least partially
259  * inside the clip_planes volume. If mode is outside, get all nodes
260  * that lie at least partially outside the volume. If showing all, get
261  * all nodes. */
262 static void get_pbvh_nodes(PBVH *pbvh,
263                            PBVHNode ***nodes,
264                            int *totnode,
265                            float clip_planes[4][4],
266                            PartialVisArea mode)
267 {
268         BLI_pbvh_SearchCallback cb = NULL;
269
270         /* select search callback */
271         switch (mode) {
272                 case PARTIALVIS_INSIDE:
273                         cb = BLI_pbvh_node_planes_contain_AABB;
274                         break;
275                 case PARTIALVIS_OUTSIDE:
276                         cb = BLI_pbvh_node_planes_exclude_AABB;
277                         break;
278                 case PARTIALVIS_ALL:
279                 case PARTIALVIS_MASKED:
280                         break;
281         }
282         
283         BLI_pbvh_search_gather(pbvh, cb, clip_planes, nodes, totnode);
284 }
285
286 static int hide_show_exec(bContext *C, wmOperator *op)
287 {
288         ARegion *ar = CTX_wm_region(C);
289         Object *ob = CTX_data_active_object(C);
290         Mesh *me = ob->data;
291         PartialVisAction action;
292         PartialVisArea area;
293         PBVH *pbvh;
294         PBVHNode **nodes;
295         DerivedMesh *dm;
296         PBVHType pbvh_type;
297         float clip_planes[4][4];
298         rcti rect;
299         int totnode, i;
300
301         /* read operator properties */
302         action = RNA_enum_get(op->ptr, "action");
303         area = RNA_enum_get(op->ptr, "area");
304         rect_from_props(&rect, op->ptr);
305
306         clip_planes_from_rect(C, clip_planes, &rect);
307
308         dm = mesh_get_derived_final(CTX_data_scene(C), ob, CD_MASK_BAREMESH);
309         pbvh = dm->getPBVH(ob, dm);
310         ob->sculpt->pbvh = pbvh;
311
312         get_pbvh_nodes(pbvh, &nodes, &totnode, clip_planes, area);
313         pbvh_type = BLI_pbvh_type(pbvh);
314
315         /* start undo */
316         switch (action) {
317                 case PARTIALVIS_HIDE:
318                         sculpt_undo_push_begin("Hide area");
319                         break;
320                 case PARTIALVIS_SHOW:
321                         sculpt_undo_push_begin("Show area");
322                         break;
323         }
324
325         for (i = 0; i < totnode; i++) {
326                 switch (pbvh_type) {
327                         case PBVH_FACES:
328                                 partialvis_update_mesh(ob, pbvh, nodes[i], action, area, clip_planes);
329                                 break;
330                         case PBVH_GRIDS:
331                                 partialvis_update_grids(ob, pbvh, nodes[i], action, area, clip_planes);
332                                 break;
333                 }
334         }
335
336         if (nodes)
337                 MEM_freeN(nodes);
338         
339         /* end undo */
340         sculpt_undo_push_end();
341
342         /* ensure that edges and faces get hidden as well (not used by
343          * sculpt but it looks wrong when entering editmode otherwise) */
344         if (pbvh_type == PBVH_FACES) {
345                 BKE_mesh_flush_hidden_from_verts(me->mvert, me->mloop,
346                                                  me->medge, me->totedge,
347                                                  me->mpoly, me->totpoly);
348         }
349
350         ED_region_tag_redraw(ar);
351         
352         return OPERATOR_FINISHED;
353 }
354
355 static int hide_show_invoke(bContext *C, wmOperator *op, wmEvent *event)
356 {
357         PartialVisArea area = RNA_enum_get(op->ptr, "area");
358
359         if (!ELEM(area, PARTIALVIS_ALL, PARTIALVIS_MASKED))
360                 return WM_border_select_invoke(C, op, event);
361         else
362                 return op->type->exec(C, op);
363 }
364
365 void PAINT_OT_hide_show(struct wmOperatorType *ot)
366 {
367         static EnumPropertyItem action_items[] = {
368                 {PARTIALVIS_HIDE, "HIDE", 0, "Hide", "Hide vertices"},
369                 {PARTIALVIS_SHOW, "SHOW", 0, "Show", "Show vertices"},
370                 {0, NULL, 0, NULL, NULL}
371         };
372
373         static EnumPropertyItem area_items[] = {
374                 {PARTIALVIS_OUTSIDE, "OUTSIDE", 0, "Outside", "Hide or show vertices outside the selection"},
375                 {PARTIALVIS_INSIDE, "INSIDE", 0, "Inside", "Hide or show vertices inside the selection"},
376                 {PARTIALVIS_ALL, "ALL", 0, "All", "Hide or show all vertices"},
377                 {PARTIALVIS_MASKED, "MASKED", 0, "Masked", "Hide or show vertices that are masked (minimum mask value of 0.5)"},
378                 {0, NULL, 0, NULL, NULL}
379         };
380         
381         /* identifiers */
382         ot->name = "Hide/Show";
383         ot->idname = "PAINT_OT_hide_show";
384         ot->description = "Hide/show some vertices";
385
386         /* api callbacks */
387         ot->invoke = hide_show_invoke;
388         ot->modal = WM_border_select_modal;
389         ot->exec = hide_show_exec;
390         /* sculpt-only for now */
391         ot->poll = sculpt_mode_poll;
392
393         ot->flag = OPTYPE_REGISTER;
394
395         /* rna */
396         RNA_def_enum(ot->srna, "action", action_items, PARTIALVIS_HIDE,
397                      "Action", "Whether to hide or show vertices");
398         RNA_def_enum(ot->srna, "area", area_items, PARTIALVIS_INSIDE,
399                      "Area", "Which vertices to hide or show");
400         
401         RNA_def_int(ot->srna, "xmin", 0, INT_MIN, INT_MAX, "X Min", "", INT_MIN, INT_MAX);
402         RNA_def_int(ot->srna, "xmax", 0, INT_MIN, INT_MAX, "X Max", "", INT_MIN, INT_MAX);
403         RNA_def_int(ot->srna, "ymin", 0, INT_MIN, INT_MAX, "Y Min", "", INT_MIN, INT_MAX);
404         RNA_def_int(ot->srna, "ymax", 0, INT_MIN, INT_MAX, "Y Max", "", INT_MIN, INT_MAX);
405 }