fix [#36923] Merge / Delete vertices crashes for some meshes
authorCampbell Barton <ideasman42@gmail.com>
Thu, 3 Oct 2013 07:06:56 +0000 (07:06 +0000)
committerCampbell Barton <ideasman42@gmail.com>
Thu, 3 Oct 2013 07:06:56 +0000 (07:06 +0000)
add BM_ITER_MESH_MUTABLE which steps before entering the for() loop body and prevents the assert from complaining about removing mesh data while iterating as well as the crash.

this was done in quite a few areas, more may turn up.

source/blender/bmesh/intern/bmesh_construct.c
source/blender/bmesh/intern/bmesh_iterators.c
source/blender/bmesh/intern/bmesh_iterators.h
source/blender/bmesh/operators/bmo_dissolve.c
source/blender/bmesh/operators/bmo_join_triangles.c
source/blender/bmesh/tools/bmesh_bevel.c
source/blender/bmesh/tools/bmesh_decimate_unsubdivide.c

index 930964667e91aff3e3b7404b3ebc9cb0ca2fa047..1a7464ce340c66eb06e0ca3a5ee1ed5eac27ae4b 100644 (file)
@@ -500,10 +500,10 @@ BMFace *BM_face_create_ngon_vcloud(BMesh *bm, BMVert **vert_arr, int len,
  */
 void BMO_remove_tagged_faces(BMesh *bm, const short oflag)
 {
-       BMFace *f;
+       BMFace *f, *f_next;
        BMIter iter;
 
-       BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
+       BM_ITER_MESH_MUTABLE (f, f_next, &iter, bm, BM_FACES_OF_MESH) {
                if (BMO_elem_flag_test(bm, f, oflag)) {
                        BM_face_kill(bm, f);
                }
@@ -512,10 +512,10 @@ void BMO_remove_tagged_faces(BMesh *bm, const short oflag)
 
 void BMO_remove_tagged_edges(BMesh *bm, const short oflag)
 {
-       BMEdge *e;
+       BMEdge *e, *e_next;
        BMIter iter;
 
-       BM_ITER_MESH (e, &iter, bm, BM_EDGES_OF_MESH) {
+       BM_ITER_MESH_MUTABLE (e, e_next, &iter, bm, BM_EDGES_OF_MESH) {
                if (BMO_elem_flag_test(bm, e, oflag)) {
                        BM_edge_kill(bm, e);
                }
@@ -524,10 +524,10 @@ void BMO_remove_tagged_edges(BMesh *bm, const short oflag)
 
 void BMO_remove_tagged_verts(BMesh *bm, const short oflag)
 {
-       BMVert *v;
+       BMVert *v, *v_next;
        BMIter iter;
 
-       BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+       BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
                if (BMO_elem_flag_test(bm, v, oflag)) {
                        BM_vert_kill(bm, v);
                }
index cc5e4933d39119a83af5b083628721db17b37831..6449a6e6d352954b7d8cce23e93072080defe60d 100644 (file)
@@ -53,6 +53,31 @@ const char bm_iter_itype_htype_map[BM_ITYPE_MAX] = {
        BM_LOOP  /* BM_LOOPS_OF_EDGE */
 };
 
+/**
+ * Utility function.
+ */
+int BM_iter_mesh_count(BMesh *bm, const char itype)
+{
+       int count;
+
+       switch (itype) {
+               case BM_VERTS_OF_MESH:
+                       count = bm->totvert;
+                       break;
+               case BM_EDGES_OF_MESH:
+                       count = bm->totedge;
+                       break;
+               case BM_FACES_OF_MESH:
+                       count = bm->totface;
+                       break;
+               default:
+                       count = 0;
+                       BLI_assert(0);
+       }
+
+       return count;
+}
+
 /**
  * \note Use #BM_vert_at_index / #BM_edge_at_index / #BM_face_at_index for mesh arrays.
  */
index bb25d3fabd161b850b0d233c27e447d62b47034b..fdf0f27f05fccb5818f493f8f8dfb29220b9b042 100644 (file)
@@ -89,6 +89,20 @@ extern const char bm_iter_itype_htype_map[BM_ITYPE_MAX];
 #define BM_ITER_MESH_INDEX(ele, iter, bm, itype, indexvar) \
        for (ele = BM_iter_new(iter, bm, itype, NULL), indexvar = 0; ele; ele = BM_iter_step(iter), (indexvar)++)
 
+/* a version of BM_ITER_MESH which keeps the next item in storage
+ * so we can delete the current item, see bug [#36923] */
+#ifdef DEBUG
+#  define BM_ITER_MESH_MUTABLE(ele, ele_next, iter, bm, itype) \
+       for (ele = BM_iter_new(iter, bm, itype, NULL); \
+       ele ? ((void)((iter)->count = BM_iter_mesh_count(bm, itype)), \
+              (void)(ele_next = BM_iter_step(iter)), 1) : 0; \
+       ele = ele_next)
+#else
+#  define BM_ITER_MESH_MUTABLE(ele, ele_next, iter, bm, itype) \
+       for (ele = BM_iter_new(iter, bm, itype, NULL); ele ? ((ele_next = BM_iter_step(iter)), 1) : 0; ele = ele_next)
+#endif
+
+
 #define BM_ITER_ELEM(ele, iter, data, itype) \
        for (ele = BM_iter_new(iter, NULL, itype, data); ele; ele = BM_iter_step(iter))
 
@@ -182,6 +196,7 @@ typedef struct BMIter {
        char itype;
 } BMIter;
 
+int     BM_iter_mesh_count(BMesh *bm, const char itype);
 void   *BM_iter_at_index(BMesh *bm, const char itype, void *data, int index) ATTR_WARN_UNUSED_RESULT;
 int     BM_iter_as_array(BMesh *bm, const char itype, void *data, void **array, const int len);
 void   *BM_iter_as_arrayN(BMesh *bm, const char itype, void *data, int *r_len,
index d3c7a6864c33550cfede53c80955a529768e8bfd..e85a9c081b9841d55882c48ae6d1ddf4844b9543 100644 (file)
@@ -293,7 +293,8 @@ void bmo_dissolve_edges_exec(BMesh *bm, BMOperator *op)
        }
 
        if (use_verts) {
-               BM_ITER_MESH (v, &viter, bm, BM_VERTS_OF_MESH) {
+               BMVert *v_next;
+               BM_ITER_MESH_MUTABLE (v, v_next, &viter, bm, BM_VERTS_OF_MESH) {
                        if (BMO_elem_flag_test(bm, v, VERT_MARK)) {
                                if (BM_vert_edge_count(v) == 2) {
                                        BM_vert_collapse_edge(bm, v->e, v, true);
@@ -349,7 +350,7 @@ static bool test_extra_verts(BMesh *bm, BMVert *v)
 void bmo_dissolve_verts_exec(BMesh *bm, BMOperator *op)
 {
        BMIter iter, fiter;
-       BMVert *v;
+       BMVert *v, *v_next;
        BMFace *f;
 
        const bool use_face_split = BMO_slot_bool_get(op->slots_in, "use_face_split");
@@ -409,7 +410,7 @@ void bmo_dissolve_verts_exec(BMesh *bm, BMOperator *op)
        }
        
        /* clean up any remainin */
-       BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+       BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
                if (BMO_elem_flag_test(bm, v, VERT_MARK)) {
                        if (!BM_vert_dissolve(bm, v)) {
                                BMO_error_raise(bm, op, BMERR_DISSOLVEVERTS_FAILED, NULL);
index 6d4f257909cd244aea4b623723e4c931a84b2e66..76678bfa4f7f8959dc78b907eccfd536f0ecf179 100644 (file)
@@ -203,7 +203,7 @@ void bmo_join_triangles_exec(BMesh *bm, BMOperator *op)
        BMIter iter;
        BMOIter siter;
        BMFace *f;
-       BMEdge *e;
+       BMEdge *e, *e_next;
        /* data: edge-to-join, sort_value: error weight */
        struct SortPointerByFloat *jedges;
        unsigned i, totedge;
@@ -291,7 +291,7 @@ void bmo_join_triangles_exec(BMesh *bm, BMOperator *op)
        MEM_freeN(jedges);
 
        /* join best weighted */
-       BM_ITER_MESH (e, &iter, bm, BM_EDGES_OF_MESH) {
+       BM_ITER_MESH_MUTABLE (e, e_next, &iter, bm, BM_EDGES_OF_MESH) {
                BMFace *f_new;
                BMFace *f_a, *f_b;
 
index 9db751ad1b7042fd2ad18e0d0e0a6f89173903dd..8c3edfa23c43ed0398ad9d4589a987bd2c2963b7 100644 (file)
@@ -2375,7 +2375,7 @@ void BM_mesh_bevel(BMesh *bm, const float offset, const float segments,
                    const struct MDeformVert *dvert, const int vertex_group)
 {
        BMIter iter;
-       BMVert *v;
+       BMVert *v, *v_next;
        BMEdge *e;
        BevelParams bp = {NULL};
 
@@ -2418,7 +2418,7 @@ void BM_mesh_bevel(BMesh *bm, const float offset, const float segments,
                        }
                }
 
-               BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+               BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
                        if (BM_elem_flag_test(v, BM_ELEM_TAG)) {
                                BLI_assert(find_bevvert(&bp, v) != NULL);
                                BM_vert_kill(bm, v);
index bc9288f98f6523e457279da2cd35ec3825ed97e1..868caa49ec75e1273883b97ccad32d0b2365503e 100644 (file)
@@ -197,7 +197,7 @@ void BM_mesh_decimate_unsubdivide_ex(BMesh *bm, const int iterations, const bool
        }
 
        for (iter_step = 0; iter_step < iterations; iter_step++) {
-               BMVert *v;
+               BMVert *v, *v_next;
                bool iter_done;
 
                BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
@@ -318,7 +318,7 @@ void BM_mesh_decimate_unsubdivide_ex(BMesh *bm, const int iterations, const bool
 
                /* now we tagged all verts -1 for removal, lets loop over and rebuild faces */
                iter_done = false;
-               BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+               BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
                        if (BM_elem_index_get(v) == VERT_INDEX_DO_COLLAPSE) {
                                if (bm_vert_dissolve_fan(bm, v)) {
                                        iter_done = true;