Fix #30720: Creating Navmesh crashes blender
authorSergey Sharybin <sergey.vfx@gmail.com>
Mon, 16 Apr 2012 13:53:30 +0000 (13:53 +0000)
committerSergey Sharybin <sergey.vfx@gmail.com>
Mon, 16 Apr 2012 13:53:30 +0000 (13:53 +0000)
In fact there were several issues fixed (all of them regressions since bmesh merge):

- Creating navmesh crashed because creating new faces for mesh was trying to set
  default values for all customdata layers in this face. This requires memory
  pool created for this datablock.
  Usually this pool is creating on creating datablock if there're some elements
  to be stored in this block. In cases of regular primitive creating it wasn't
  an issue because they doesn't create customdata layers, they only creates
  geometry.
  Navigation mesh creates geometry and customdata layers (CD_RECAST layer)
  which used to confuse a bit custom data functions. Solved by ensuring there's
  memory pool created for polygons datablock after adding new custom data layer.
  Most probably it's better to be resolved on CD level (like smarter track on
  changed amount of stored data and so) but prefer not to make such global changes
  so close to the release.
- Toggling edit mode lead to loosing recast datalayer. Solved by adding recast
  layer to bmesh mask so it'll be copied to/from edit mesh.
- Some part of code assumed raycast layer is in face datablock, some that it's in
  polygon datablock. Made it to be in polygons datablock.
  Kind of temporary solution to make navmesh working, probably it'll fail if one
  will want to edit navmesh by hand after it was generated.
  Proper way would be to ensure the whole navmesh things are using ngons.

source/blender/blenkernel/intern/DerivedMesh.c
source/blender/blenkernel/intern/customdata.c
source/blender/blenkernel/intern/navmesh_conversion.c
source/blender/editors/mesh/mesh_navmesh.c
source/gameengine/Ketsji/KX_NavMeshObject.cpp

index e6fb506620c742689b0ccb8bedb5380276ebbdf4..3330a6596a78830468fdfc905bf399735bbd842c 100644 (file)
@@ -1753,18 +1753,6 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
                        add_orco_dm(ob, NULL, *deform_r, NULL, CD_ORCO);
        }
 
-#ifdef WITH_GAMEENGINE
-       /* NavMesh - this is a hack but saves having a NavMesh modifier */
-       if ((ob->gameflag & OB_NAVMESH) && (finaldm->type == DM_TYPE_CDDM)) {
-               DerivedMesh *tdm;
-               tdm= navmesh_dm_createNavMeshForVisualization(finaldm);
-               if (finaldm != tdm) {
-                       finaldm->release(finaldm);
-                       finaldm= tdm;
-               }
-       }
-#endif /* WITH_GAMEENGINE */
-
        {
                /* calculating normals can re-calculate tessfaces in some cases */
 #if 0
@@ -1820,6 +1808,18 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
                }
        }
 
+#ifdef WITH_GAMEENGINE
+       /* NavMesh - this is a hack but saves having a NavMesh modifier */
+       if ((ob->gameflag & OB_NAVMESH) && (finaldm->type == DM_TYPE_CDDM)) {
+               DerivedMesh *tdm;
+               tdm= navmesh_dm_createNavMeshForVisualization(finaldm);
+               if (finaldm != tdm) {
+                       finaldm->release(finaldm);
+                       finaldm= tdm;
+               }
+       }
+#endif /* WITH_GAMEENGINE */
+
        *final_r = finaldm;
 
        if (orcodm)
@@ -2898,7 +2898,7 @@ static void navmesh_drawColored(DerivedMesh *dm)
        int a, glmode;
        MVert *mvert = (MVert *)CustomData_get_layer(&dm->vertData, CD_MVERT);
        MFace *mface = (MFace *)CustomData_get_layer(&dm->faceData, CD_MFACE);
-       int *polygonIdx = (int *)CustomData_get_layer(&dm->faceData, CD_RECAST);
+       int *polygonIdx = (int *)CustomData_get_layer(&dm->polyData, CD_RECAST);
        float col[3];
 
        if (!polygonIdx)
@@ -2980,14 +2980,14 @@ static DerivedMesh *navmesh_dm_createNavMeshForVisualization(DerivedMesh *dm)
        int res;
 
        result = CDDM_copy(dm);
-       if (!CustomData_has_layer(&result->faceData, CD_RECAST)) {
-               int *sourceRecastData = (int*)CustomData_get_layer(&dm->faceData, CD_RECAST);
+       if (!CustomData_has_layer(&result->polyData, CD_RECAST)) {
+               int *sourceRecastData = (int*)CustomData_get_layer(&dm->polyData, CD_RECAST);
                if (sourceRecastData) {
-                       CustomData_add_layer_named(&result->faceData, CD_RECAST, CD_DUPLICATE,
+                       CustomData_add_layer_named(&result->polyData, CD_RECAST, CD_DUPLICATE,
                                                   sourceRecastData, maxFaces, "recastData");
                }
        }
-       recastData = (int*)CustomData_get_layer(&result->faceData, CD_RECAST);
+       recastData = (int*)CustomData_get_layer(&result->polyData, CD_RECAST);
 
        /* note: This is not good design! - really should not be doing this */
        result->drawFacesTex =  navmesh_DM_drawFacesTex;
index f0bda57466d9a55786faeff1075173cd6bedfae9..2450f3ca83ed65ef3ddb068dd4c23984548047a3 100644 (file)
@@ -1099,7 +1099,7 @@ const CustomDataMask CD_MASK_DERIVEDMESH =
        CD_MASK_ORIGINDEX | CD_MASK_POLYINDEX;
 const CustomDataMask CD_MASK_BMESH = CD_MASK_MLOOPUV | CD_MASK_MLOOPCOL | CD_MASK_MTEXPOLY |
        CD_MASK_MSTICKY | CD_MASK_MDEFORMVERT | CD_MASK_PROP_FLT | CD_MASK_PROP_INT | 
-       CD_MASK_PROP_STR | CD_MASK_SHAPEKEY | CD_MASK_SHAPE_KEYINDEX | CD_MASK_MDISPS | CD_MASK_CREASE | CD_MASK_BWEIGHT;
+       CD_MASK_PROP_STR | CD_MASK_SHAPEKEY | CD_MASK_SHAPE_KEYINDEX | CD_MASK_MDISPS | CD_MASK_CREASE | CD_MASK_BWEIGHT | CD_MASK_RECAST;
 const CustomDataMask CD_MASK_FACECORNERS =
        CD_MASK_MTFACE | CD_MASK_MCOL | CD_MASK_MTEXPOLY | CD_MASK_MLOOPUV |
        CD_MASK_MLOOPCOL;
index 27e309e9d20a11e649cf6abe53cc748e3e3697cb..34e0be1de9244b6e6237c449155e1dfc39436e15 100644 (file)
@@ -166,7 +166,7 @@ int buildRawVertIndicesData(DerivedMesh* dm, int *nverts_r, float **verts_r,
        }
 
        //carefully, recast data is just reference to data in derived mesh
-       *recastData = (int*)CustomData_get_layer(&dm->faceData, CD_RECAST);
+       *recastData = (int*)CustomData_get_layer(&dm->polyData, CD_RECAST);
 
        *nverts_r = nverts;
        *verts_r = verts;
index f1b0a82b654a859c65a8e0ae1b612d87f10c1e9f..7d78a527b0619b9051f33599c39aa8f658f22636 100644 (file)
@@ -345,6 +345,7 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
 
        /* create custom data layer to save polygon idx */
        CustomData_add_layer_named(&em->bm->pdata, CD_RECAST, CD_CALLOC, NULL, 0, "createRepresentation recastData");
+       CustomData_bmesh_init_pool(&em->bm->pdata, 0, BM_FACE);
        
        /* create verts and faces for detailed mesh */
        meshes = recast_polyMeshDetailGetMeshes(dmesh, &nmeshes);
index 22f96eb72979121ec05fc722c6929405242efeff..35058e5fe5dbd3c372e82517e947b940f1123584 100644 (file)
@@ -112,7 +112,8 @@ bool KX_NavMeshObject::BuildVertIndArrays(float *&vertices, int& nverts,
 {
        DerivedMesh* dm = mesh_create_derived_no_virtual(KX_GetActiveScene()->GetBlenderScene(), GetBlenderObject(), 
                                                                                                        NULL, CD_MASK_MESH);
-       int* recastData = (int*) dm->getTessFaceDataArray(dm, CD_RECAST);
+       CustomData *pdata = dm->getPolyDataLayout(dm);
+       int* recastData = (int*) CustomData_get_layer(pdata, CD_RECAST);
        if (recastData)
        {
                int *dtrisToPolysMap=NULL, *dtrisToTrisMap=NULL, *trisToFacesMap=NULL;