went over all uses of MDeformWeight.def_nr and made sure the value is clamped when...
authorCampbell Barton <ideasman42@gmail.com>
Fri, 9 Dec 2011 08:20:27 +0000 (08:20 +0000)
committerCampbell Barton <ideasman42@gmail.com>
Fri, 9 Dec 2011 08:20:27 +0000 (08:20 +0000)
source/blender/blenkernel/BKE_deform.h
source/blender/blenkernel/intern/DerivedMesh.c
source/blender/blenkernel/intern/armature.c
source/blender/blenkernel/intern/deform.c
source/blender/editors/object/object_vgroup.c
source/blender/modifiers/intern/MOD_mask.c
source/gameengine/Converter/BL_SkinDeformer.cpp

index b08afd9c475cf6d19f6354c18c7f30eec3adfd33..9902b26e15bf7c282ee6759c4294768a3b514f56 100644 (file)
@@ -63,7 +63,7 @@ void defvert_copy_index(struct MDeformVert *dvert_dst, const struct MDeformVert
 void defvert_sync(struct MDeformVert *dvert_dst, const struct MDeformVert *dvert_src, int use_verify);
 void defvert_sync_mapped(struct MDeformVert *dvert_dst, const struct MDeformVert *dvert_src,
                          const int *flip_map, const int flip_map_len, const int use_verify);
-void defvert_remap (struct MDeformVert *dvert, int *map);
+void defvert_remap (struct MDeformVert *dvert, int *map, const int map_len);
 void defvert_flip(struct MDeformVert *dvert, const int *flip_map, const int flip_map_len);
 void defvert_normalize(struct MDeformVert *dvert);
 
index 1498a3b4cc08129844f2e0621a09d54ed773fcd5..4b91911a6ab6f17b3b41c63a815f0837ef4a17ca 100644 (file)
@@ -642,7 +642,7 @@ enum {
 };
 
 static void calc_weightpaint_vert_color(
-        Object *ob, ColorBand *coba, int vert, unsigned char *col,
+        Object *ob, const int defbase_tot, ColorBand *coba, int vert, unsigned char *col,
         const char *dg_flags, int selected, int UNUSED(unselected), const int draw_flag)
 {
        Mesh *me = ob->data;
@@ -661,10 +661,12 @@ static void calc_weightpaint_vert_color(
                        for (i = dvert->totweight; i > 0; i--, dw++) {
                                /* in multipaint, get the average if auto normalize is inactive
                                 * get the sum if it is active */
-                               if (dg_flags[dw->def_nr]) {
-                                       if (dw->weight) {
-                                               input += dw->weight;
-                                               was_a_nonzero= TRUE;
+                               if (dw->def_nr < defbase_tot) {
+                                       if (dg_flags[dw->def_nr]) {
+                                               if (dw->weight) {
+                                                       input += dw->weight;
+                                                       was_a_nonzero= TRUE;
+                                               }
                                        }
                                }
                        }
@@ -718,20 +720,20 @@ static void add_weight_mcol_dm(Object *ob, DerivedMesh *dm, int const draw_flag)
        unsigned char *wtcol;
        int i;
        
-       int defbase_len = BLI_countlist(&ob->defbase);
-       char *defbase_sel = MEM_mallocN(defbase_len * sizeof(char), __func__);
-       int selected = get_selected_defgroups(ob, defbase_sel, defbase_len);
-       int unselected = defbase_len - selected;
+       int defbase_tot = BLI_countlist(&ob->defbase);
+       char *defbase_sel = MEM_mallocN(defbase_tot * sizeof(char), __func__);
+       int selected = get_selected_defgroups(ob, defbase_sel, defbase_tot);
+       int unselected = defbase_tot - selected;
 
        wtcol = MEM_callocN (sizeof (unsigned char) * me->totface*4*4, "weightmap");
        
        memset(wtcol, 0x55, sizeof (unsigned char) * me->totface*4*4);
        for (i=0; i<me->totface; i++, mf++) {
-               calc_weightpaint_vert_color(ob, coba, mf->v1, &wtcol[(i*4 + 0)*4], defbase_sel, selected, unselected, draw_flag);
-               calc_weightpaint_vert_color(ob, coba, mf->v2, &wtcol[(i*4 + 1)*4], defbase_sel, selected, unselected, draw_flag);
-               calc_weightpaint_vert_color(ob, coba, mf->v3, &wtcol[(i*4 + 2)*4], defbase_sel, selected, unselected, draw_flag);
+               calc_weightpaint_vert_color(ob, defbase_tot, coba, mf->v1, &wtcol[(i*4 + 0)*4], defbase_sel, selected, unselected, draw_flag);
+               calc_weightpaint_vert_color(ob, defbase_tot, coba, mf->v2, &wtcol[(i*4 + 1)*4], defbase_sel, selected, unselected, draw_flag);
+               calc_weightpaint_vert_color(ob, defbase_tot, coba, mf->v3, &wtcol[(i*4 + 2)*4], defbase_sel, selected, unselected, draw_flag);
                if (mf->v4)
-                       calc_weightpaint_vert_color(ob, coba, mf->v4, &wtcol[(i*4 + 3)*4], defbase_sel, selected, unselected, draw_flag);
+                       calc_weightpaint_vert_color(ob, defbase_tot, coba, mf->v4, &wtcol[(i*4 + 3)*4], defbase_sel, selected, unselected, draw_flag);
        }
        
        MEM_freeN(defbase_sel);
index e7743bea2cdd7e0746c60485b02f3e5b6cbf4006..e10c4b244580d90deda8a660160b8220b6ffa953 100644 (file)
@@ -827,7 +827,7 @@ void armature_deform_verts(Object *armOb, Object *target, DerivedMesh *dm,
        const short use_envelope = deformflag & ARM_DEF_ENVELOPE;
        const short use_quaternion = deformflag & ARM_DEF_QUATERNION;
        const short invert_vgroup= deformflag & ARM_DEF_INVERT_VGROUP;
-       int numGroups = 0;              /* safety for vertexgroup index overflow */
+       int defbase_tot = 0;            /* safety for vertexgroup index overflow */
        int i, target_totvert = 0;      /* safety for vertexgroup overflow */
        int use_dverts = 0;
        int armature_def_nr;
@@ -869,7 +869,7 @@ void armature_deform_verts(Object *armOb, Object *target, DerivedMesh *dm,
        armature_def_nr= defgroup_name_index(target, defgrp_name);
        
        if(ELEM(target->type, OB_MESH, OB_LATTICE)) {
-               numGroups = BLI_countlist(&target->defbase);
+               defbase_tot = BLI_countlist(&target->defbase);
                
                if(target->type==OB_MESH) {
                        Mesh *me= target->data;
@@ -896,8 +896,8 @@ void armature_deform_verts(Object *armOb, Object *target, DerivedMesh *dm,
                        else if(dverts) use_dverts = 1;
 
                        if(use_dverts) {
-                               defnrToPC = MEM_callocN(sizeof(*defnrToPC) * numGroups, "defnrToBone");
-                               defnrToPCIndex = MEM_callocN(sizeof(*defnrToPCIndex) * numGroups, "defnrToIndex");
+                               defnrToPC = MEM_callocN(sizeof(*defnrToPC) * defbase_tot, "defnrToBone");
+                               defnrToPCIndex = MEM_callocN(sizeof(*defnrToPCIndex) * defbase_tot, "defnrToIndex");
                                for(i = 0, dg = target->defbase.first; dg;
                                        i++, dg = dg->next) {
                                        defnrToPC[i] = get_pose_channel(armOb->pose, dg->name);
@@ -975,7 +975,7 @@ void armature_deform_verts(Object *armOb, Object *target, DerivedMesh *dm,
                        
                        for(j = 0; j < dvert->totweight; j++){
                                int index = dvert->dw[j].def_nr;
-                               if(index < numGroups && (pchan= defnrToPC[index])) {
+                               if(index < defbase_tot && (pchan= defnrToPC[index])) {
                                        float weight = dvert->dw[j].weight;
                                        Bone *bone= pchan->bone;
                                        pdef_info= pdef_info_array + defnrToPCIndex[index];
index e5176663228e28a5f169a917875141bce28e0ce3..94be15e27c0c7c96486c8d212178f1ff7042e0ee 100644 (file)
@@ -161,12 +161,14 @@ void defvert_sync_mapped(MDeformVert *dvert_dst, const MDeformVert *dvert_src,
 }
 
 /* be sure all flip_map values are valid */
-void defvert_remap(MDeformVert *dvert, int *map)
+void defvert_remap(MDeformVert *dvert, int *map, const int map_len)
 {
        MDeformWeight *dw;
        int i;
        for (i=0, dw=dvert->dw; i<dvert->totweight; i++, dw++) {
-               dw->def_nr= map[dw->def_nr];
+               if (dw->def_nr < map_len) {
+                       dw->def_nr= map[dw->def_nr];
+               }
        }
 }
 
@@ -198,8 +200,10 @@ void defvert_flip(MDeformVert *dvert, const int *flip_map, const int flip_map_le
        int i;
 
        for (dw= dvert->dw, i=0; i<dvert->totweight; dw++, i++) {
-               if ((dw->def_nr < flip_map_len) && (flip_map[dw->def_nr] >= 0)) {
-                       dw->def_nr= flip_map[dw->def_nr];
+               if (dw->def_nr < flip_map_len) {
+                       if (flip_map[dw->def_nr] >= 0) {
+                               dw->def_nr= flip_map[dw->def_nr];
+                       }
                }
        }
 }
@@ -283,17 +287,17 @@ int defgroup_find_index(Object *ob, bDeformGroup *dg)
 /* note, must be freed */
 int *defgroup_flip_map(Object *ob, int *flip_map_len, int use_default)
 {
-       int totdg= *flip_map_len= BLI_countlist(&ob->defbase);
+       int defbase_tot= *flip_map_len= BLI_countlist(&ob->defbase);
 
-       if (totdg==0) {
+       if (defbase_tot==0) {
                return NULL;
        }
        else {
                bDeformGroup *dg;
                char name[sizeof(dg->name)];
-               int i, flip_num, *map= MEM_mallocN(totdg * sizeof(int), __func__);
+               int i, flip_num, *map= MEM_mallocN(defbase_tot * sizeof(int), __func__);
 
-               for (i=0; i < totdg; i++) {
+               for (i=0; i < defbase_tot; i++) {
                        map[i]= -1;
                }
 
@@ -321,17 +325,17 @@ int *defgroup_flip_map(Object *ob, int *flip_map_len, int use_default)
 /* note, must be freed */
 int *defgroup_flip_map_single(Object *ob, int *flip_map_len, int use_default, int defgroup)
 {
-       int totdg= *flip_map_len= BLI_countlist(&ob->defbase);
+       int defbase_tot= *flip_map_len= BLI_countlist(&ob->defbase);
 
-       if (totdg==0) {
+       if (defbase_tot==0) {
                return NULL;
        }
        else {
                bDeformGroup *dg;
                char name[sizeof(dg->name)];
-               int i, flip_num, *map= MEM_mallocN(totdg * sizeof(int), __func__);
+               int i, flip_num, *map= MEM_mallocN(defbase_tot * sizeof(int), __func__);
 
-               for (i=0; i < totdg; i++) {
+               for (i=0; i < defbase_tot; i++) {
                        if (use_default) map[i]= i;
                        else             map[i]= -1;
                }
index 3401157359edafb1f6db6c913029d0f469cf4d5f..408558a30eb87140395297c1f95dc7052ef27d56 100644 (file)
@@ -286,8 +286,8 @@ int ED_vgroup_copy_array(Object *ob, Object *ob_from)
        int dvert_tot_from;
        int dvert_tot;
        int i;
-       int totdef_from= BLI_countlist(&ob_from->defbase);
-       int totdef= BLI_countlist(&ob->defbase);
+       int defbase_tot_from= BLI_countlist(&ob_from->defbase);
+       int defbase_tot= BLI_countlist(&ob->defbase);
        short new_vgroup= FALSE;
 
        ED_vgroup_give_parray(ob_from->data, &dvert_array_from, &dvert_tot_from);
@@ -314,11 +314,11 @@ int ED_vgroup_copy_array(Object *ob, Object *ob_from)
        BLI_duplicatelist(&ob->defbase, &ob_from->defbase);
        ob->actdef= ob_from->actdef;
 
-       if(totdef_from < totdef) {
+       if(defbase_tot_from < defbase_tot) {
                /* correct vgroup indices because the number of vgroups is being reduced. */
-               int *remap= MEM_mallocN(sizeof(int) * (totdef + 1), "ED_vgroup_copy_array");
-               for(i=0; i<=totdef_from; i++) remap[i]= i;
-               for(; i<=totdef; i++) remap[i]= 0; /* can't use these, so disable */
+               int *remap= MEM_mallocN(sizeof(int) * (defbase_tot + 1), "ED_vgroup_copy_array");
+               for(i=0; i<=defbase_tot_from; i++) remap[i]= i;
+               for(; i<=defbase_tot; i++) remap[i]= 0; /* can't use these, so disable */
 
                vgroup_remap_update_users(ob, remap);
                MEM_freeN(remap);
@@ -1809,12 +1809,12 @@ static void vgroup_remap_update_users(Object *ob, int *map)
 
 static void vgroup_delete_update_users(Object *ob, int id)
 {
-       int i, tot= BLI_countlist(&ob->defbase) + 1;
-       int *map= MEM_mallocN(sizeof(int) * tot, "vgroup del");
+       int i, defbase_tot= BLI_countlist(&ob->defbase) + 1;
+       int *map= MEM_mallocN(sizeof(int) * defbase_tot, "vgroup del");
 
        map[id]= map[0]= 0;
        for(i=1; i<id; i++) map[i]=i;
-       for(i=id+1; i<tot; i++) map[i]=i-1;
+       for(i=id+1; i<defbase_tot; i++) map[i]=i-1;
 
        vgroup_remap_update_users(ob, map);
        MEM_freeN(map);
@@ -2817,8 +2817,8 @@ void OBJECT_OT_vertex_group_set_active(wmOperatorType *ot)
 static char *vgroup_init_remap(Object *ob)
 {
        bDeformGroup *def;
-       int def_tot = BLI_countlist(&ob->defbase);
-       char *name_array= MEM_mallocN(MAX_VGROUP_NAME * sizeof(char) * def_tot, "sort vgroups");
+       int defbase_tot = BLI_countlist(&ob->defbase);
+       char *name_array= MEM_mallocN(MAX_VGROUP_NAME * sizeof(char) * defbase_tot, "sort vgroups");
        char *name;
 
        name= name_array;
@@ -2834,8 +2834,8 @@ static int vgroup_do_remap(Object *ob, char *name_array, wmOperator *op)
 {
        MDeformVert *dvert= NULL;
        bDeformGroup *def;
-       int def_tot = BLI_countlist(&ob->defbase);
-       int *sort_map_update= MEM_mallocN(sizeof(int) * (def_tot + 1), "sort vgroups"); /* needs a dummy index at the start*/
+       int defbase_tot = BLI_countlist(&ob->defbase);
+       int *sort_map_update= MEM_mallocN(sizeof(int) * (defbase_tot + 1), "sort vgroups"); /* needs a dummy index at the start*/
        int *sort_map= sort_map_update + 1;
        char *name;
        int i;
@@ -2856,7 +2856,7 @@ static int vgroup_do_remap(Object *ob, char *name_array, wmOperator *op)
                        for(eve=em->verts.first; eve; eve=eve->next){
                                dvert= CustomData_em_get(&em->vdata, eve->data, CD_MDEFORMVERT);
                                if(dvert && dvert->totweight){
-                                       defvert_remap(dvert, sort_map);
+                                       defvert_remap(dvert, sort_map, defbase_tot);
                                }
                        }
                }
@@ -2874,13 +2874,13 @@ static int vgroup_do_remap(Object *ob, char *name_array, wmOperator *op)
                /*create as necassary*/
                while(dvert && dvert_tot--) {
                        if(dvert->totweight)
-                               defvert_remap(dvert, sort_map);
+                               defvert_remap(dvert, sort_map, defbase_tot);
                        dvert++;
                }
        }
 
        /* update users */
-       for(i=0; i<def_tot; i++)
+       for(i=0; i<defbase_tot; i++)
                sort_map[i]++;
 
        sort_map_update[0]= 0;
index 4145dafce7b84894ecd7f39f0f5ef0b9a7ac70b8..1fbe0eef6f34635ff919df01571298ebd10fd02b 100644 (file)
@@ -137,12 +137,13 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
                bDeformGroup *def;
                char *bone_select_array;
                int bone_select_tot= 0;
+               const int defbase_tot= BLI_countlist(&ob->defbase);
                
                /* check that there is armature object with bones to use, otherwise return original mesh */
                if (ELEM3(NULL, mmd->ob_arm, mmd->ob_arm->pose, ob->defbase.first))
                        return derivedData;
 
-               bone_select_array= MEM_mallocN(BLI_countlist(&ob->defbase) * sizeof(char), "mask array");
+               bone_select_array= MEM_mallocN(defbase_tot * sizeof(char), "mask array");
 
                for (i = 0, def = ob->defbase.first; def; def = def->next, i++)
                {
@@ -194,12 +195,12 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
                        MDeformWeight *dw= dv->dw;
                        int j;
 
-                       for (j= dv->totweight; j > 0; j--, dw++)
-                       {
-                               if (bone_select_array[dw->def_nr])
-                               {
-                                       if(dw->weight != 0.0f) {
-                                               break;
+                       for (j= dv->totweight; j > 0; j--, dw++) {
+                               if (dw->def_nr < defbase_tot) {
+                                       if (bone_select_array[dw->def_nr]) {
+                                               if(dw->weight != 0.0f) {
+                                                       break;
+                                               }
                                        }
                                }
                        }
index 815332059096d1dbeac3e6a20c8b31ae8bf497e2..0c4806f4bd8ae79f09ba84e27e47f3c4826eeb16 100644 (file)
@@ -217,14 +217,14 @@ void BL_SkinDeformer::BGEDeformVerts()
        Object *par_arma = m_armobj->GetArmatureObject();
        MDeformVert *dverts = m_bmesh->dvert;
        bDeformGroup *dg;
-       int numGroups = BLI_countlist(&m_objMesh->defbase);
+       int defbase_tot = BLI_countlist(&m_objMesh->defbase);
 
        if (!dverts)
                return;
 
        if (m_dfnrToPC == NULL)
        {
-               m_dfnrToPC = new bPoseChannel*[numGroups];
+               m_dfnrToPC = new bPoseChannel*[defbase_tot];
                int i;
                for (i=0, dg=(bDeformGroup*)m_objMesh->defbase.first;
                        dg;
@@ -260,7 +260,7 @@ void BL_SkinDeformer::BGEDeformVerts()
                {
                        int index = dvert->dw[j].def_nr;
 
-                       if (index < numGroups && (pchan=m_dfnrToPC[index]))
+                       if (index < defbase_tot && (pchan=m_dfnrToPC[index]))
                        {
                                weight = dvert->dw[j].weight;