Fix navmesh creation w/ multiple objects
[blender-staging.git] / source / blender / editors / mesh / mesh_navmesh.c
index 99947cb9ae21dadafa26b895175f4172806a2d54..740d18951dcb7cceb8497f7ba998404bf7be237c 100644 (file)
  * ***** END GPL LICENSE BLOCK *****
  */
 
-#include <math.h>
+/** \file blender/editors/mesh/mesh_navmesh.c
+ *  \ingroup edmesh
+ */
 
 #include "MEM_guardedalloc.h"
 
 #include "DNA_scene_types.h"
 #include "DNA_object_types.h"
-#include "DNA_meshdata_types.h"
-#include "DNA_modifier_types.h"
-#include "DNA_ID.h"
+#include "DNA_mesh_types.h"
+
+#include "BLI_listbase.h"
+#include "BLI_math_vector.h"
+#include "BLI_linklist.h"
 
 #include "BKE_library.h"
 #include "BKE_depsgraph.h"
 #include "BKE_context.h"
-#include "BKE_main.h"
 #include "BKE_mesh.h"
-#include "BKE_modifier.h"
 #include "BKE_scene.h"
 #include "BKE_DerivedMesh.h"
-#include "BKE_cdderivedmesh.h"
 #include "BKE_report.h"
-#include "BKE_tessmesh.h"
-
-#include "BLI_editVert.h"
-#include "BLI_listbase.h"
-#include "BLI_utildefines.h"
-#include "BLI_math_vector.h"
-#include "BLI_linklist.h"
+#include "BKE_editmesh.h"
 
 #include "ED_object.h"
 #include "ED_mesh.h"
 #include "ED_screen.h"
 
-#include "RNA_access.h"
-
 #include "WM_api.h"
 #include "WM_types.h"
 
-#include "mesh_intern.h"
 #include "recast-capi.h"
 
-static void createVertsTrisData(bContext *C, LinkNode* obs, int *nverts_r, float **verts_r, int *ntris_r, int **tris_r)
+#include "mesh_intern.h"  /* own include */
+
+
+static void createVertsTrisData(bContext *C, LinkNode *obs,
+                                int *nverts_r, float **verts_r, int *ntris_r, int **tris_r, unsigned int *r_lay)
 {
        MVert *mvert;
-       int nfaces= 0, *tri, i, curnverts, basenverts, curnfaces;
+       int nfaces = 0, *tri, i, curnverts, basenverts, curnfaces;
        MFace *mface;
        float co[3], wco[3];
        Object *ob;
        LinkNode *oblink, *dmlink;
        DerivedMesh *dm;
-       Scene* scene= CTX_data_scene(C);
-       LinkNode* dms= NULL;
+       Scene *scene = CTX_data_scene(C);
+       LinkNodePair dms_pair = {NULL,NULL};
 
        int nverts, ntris, *tris;
        float *verts;
 
-       nverts= 0;
-       ntris= 0;
+       nverts = 0;
+       ntris = 0;
 
        /* calculate number of verts and tris */
-       for(oblink= obs; oblink; oblink= oblink->next) {
-               ob= (Object*) oblink->link;
-               dm= mesh_create_derived_no_virtual(scene, ob, NULL, CD_MASK_MESH);
-               BLI_linklist_append(&dms, (void*)dm);
+       for (oblink = obs; oblink; oblink = oblink->next) {
+               ob = (Object *) oblink->link;
+               dm = mesh_create_derived_no_virtual(scene, ob, NULL, CD_MASK_MESH);
+               DM_ensure_tessface(dm);
+               BLI_linklist_append(&dms_pair, dm);
 
-               nverts+= dm->getNumVerts(dm);
-               nfaces= dm->getNumTessFaces(dm);
-               ntris+= nfaces;
+               nverts += dm->getNumVerts(dm);
+               nfaces = dm->getNumTessFaces(dm);
+               ntris += nfaces;
 
                /* resolve quad faces */
-               mface= dm->getTessFaceArray(dm);
-               for(i= 0; i<nfaces; i++) {
-                       MFace* mf= &mface[i];
-                       if(mf->v4)
-                               ntris+=1;
+               mface = dm->getTessFaceArray(dm);
+               for (i = 0; i < nfaces; i++) {
+                       MFace *mf = &mface[i];
+                       if (mf->v4)
+                               ntris += 1;
                }
+
+               *r_lay |= ob->lay;
        }
+       LinkNode *dms = dms_pair.list;
 
        /* create data */
-       verts= MEM_mallocN(sizeof(float)*3*nverts, "createVertsTrisData verts");
-       tris= MEM_mallocN(sizeof(int)*3*ntris, "createVertsTrisData faces");
+       verts = MEM_mallocN(sizeof(float) * 3 * nverts, "createVertsTrisData verts");
+       tris = MEM_mallocN(sizeof(int) * 3 * ntris, "createVertsTrisData faces");
 
-       basenverts= 0;
-       tri= tris;
-       for(oblink= obs, dmlink= dms; oblink && dmlink;
-                       oblink= oblink->next, dmlink= dmlink->next) {
-               ob= (Object*) oblink->link;
-               dm= (DerivedMesh*) dmlink->link;
+       basenverts = 0;
+       tri = tris;
+       for (oblink = obs, dmlink = dms; oblink && dmlink;
+            oblink = oblink->next, dmlink = dmlink->next)
+       {
+               ob = (Object *) oblink->link;
+               dm = (DerivedMesh *) dmlink->link;
 
-               curnverts= dm->getNumVerts(dm);
-               mvert= dm->getVertArray(dm);
+               curnverts = dm->getNumVerts(dm);
+               mvert = dm->getVertArray(dm);
 
                /* copy verts */
-               for(i= 0; i<curnverts; i++) {
-                       MVert *v= &mvert[i];
+               for (i = 0; i < curnverts; i++) {
+                       MVert *v = &mvert[i];
 
                        copy_v3_v3(co, v->co);
                        mul_v3_m4v3(wco, ob->obmat, co);
 
-                       verts[3*(basenverts+i)+0]= wco[0];
-                       verts[3*(basenverts+i)+1]= wco[2];
-                       verts[3*(basenverts+i)+2]= wco[1];
+                       verts[3 * (basenverts + i) + 0] = wco[0];
+                       verts[3 * (basenverts + i) + 1] = wco[2];
+                       verts[3 * (basenverts + i) + 2] = wco[1];
                }
 
                /* create tris */
-               curnfaces= dm->getNumTessFaces(dm);
-               mface= dm->getTessFaceArray(dm);
+               curnfaces = dm->getNumTessFaces(dm);
+               mface = dm->getTessFaceArray(dm);
 
-               for(i= 0; i<curnfaces; i++) {
-                       MFace* mf= &mface[i];
+               for (i = 0; i < curnfaces; i++) {
+                       MFace *mf = &mface[i];
 
-                       tri[0]= basenverts + mf->v1;
-                       tri[1]= basenverts + mf->v3;
-                       tri[2]= basenverts + mf->v2;
+                       tri[0] = basenverts + mf->v1;
+                       tri[1] = basenverts + mf->v3;
+                       tri[2] = basenverts + mf->v2;
                        tri += 3;
 
-                       if(mf->v4) {
-                               tri[0]= basenverts + mf->v1;
-                               tri[1]= basenverts + mf->v4;
-                               tri[2]= basenverts + mf->v3;
+                       if (mf->v4) {
+                               tri[0] = basenverts + mf->v1;
+                               tri[1] = basenverts + mf->v4;
+                               tri[2] = basenverts + mf->v3;
                                tri += 3;
                        }
                }
 
-               basenverts+= curnverts;
+               basenverts += curnverts;
        }
 
        /* release derived mesh */
-       for(dmlink= dms; dmlink; dmlink= dmlink->next) {
-               dm= (DerivedMesh*) dmlink->link;
+       for (dmlink = dms; dmlink; dmlink = dmlink->next) {
+               dm = (DerivedMesh *) dmlink->link;
                dm->release(dm);
        }
 
        BLI_linklist_free(dms, NULL);
 
-       *nverts_r= nverts;
-       *verts_r= verts;
-       *ntris_r= ntris;
-       *tris_r= tris;
+       *nverts_r = nverts;
+       *verts_r = verts;
+       *ntris_r = ntris;
+       *tris_r = tris;
 }
 
-static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
-                                                                struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh)
+static bool buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
+                         struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh,
+                         ReportList *reports)
 {
        float bmin[3], bmax[3];
        struct recast_heightfield *solid;
        unsigned char *triflags;
-       struct recast_compactHeightfieldchf;
+       struct recast_compactHeightfield *chf;
        struct recast_contourSet *cset;
        int width, height, walkableHeight, walkableClimb, walkableRadius;
        int minRegionArea, mergeRegionArea, maxEdgeLen;
@@ -181,35 +183,41 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
        recast_calcBounds(verts, nverts, bmin, bmax);
 
        /* ** Step 1. Initialize build config ** */
-       walkableHeight= (int)ceilf(recastParams->agentheight/ recastParams->cellheight);
-       walkableClimb= (int)floorf(recastParams->agentmaxclimb / recastParams->cellheight);
-       walkableRadius= (int)ceilf(recastParams->agentradius / recastParams->cellsize);
-       minRegionArea= (int)(recastParams->regionminsize * recastParams->regionminsize);
-       mergeRegionArea= (int)(recastParams->regionmergesize * recastParams->regionmergesize);
-       maxEdgeLen= (int)(recastParams->edgemaxlen/recastParams->cellsize);
-       detailSampleDist= recastParams->detailsampledist< 0.9f ? 0 :
-                       recastParams->cellsize * recastParams->detailsampledist;
-       detailSampleMaxError= recastParams->cellheight * recastParams->detailsamplemaxerror;
+       walkableHeight = (int)ceilf(recastParams->agentheight / recastParams->cellheight);
+       walkableClimb = (int)floorf(recastParams->agentmaxclimb / recastParams->cellheight);
+       walkableRadius = (int)ceilf(recastParams->agentradius / recastParams->cellsize);
+       minRegionArea = (int)(recastParams->regionminsize * recastParams->regionminsize);
+       mergeRegionArea = (int)(recastParams->regionmergesize * recastParams->regionmergesize);
+       maxEdgeLen = (int)(recastParams->edgemaxlen / recastParams->cellsize);
+       detailSampleDist = recastParams->detailsampledist < 0.9f ? 0 :
+                          recastParams->cellsize * recastParams->detailsampledist;
+       detailSampleMaxError = recastParams->cellheight * recastParams->detailsamplemaxerror;
 
        /* Set the area where the navigation will be build. */
        recast_calcGridSize(bmin, bmax, recastParams->cellsize, &width, &height);
 
+       /* zero dimensions cause zero alloc later on [#33758] */
+       if (width <= 0 || height <= 0) {
+               BKE_report(reports, RPT_ERROR, "Object has a width or height of zero");
+               return false;
+       }
+
        /* ** Step 2: Rasterize input polygon soup ** */
        /* Allocate voxel heightfield where we rasterize our input data to */
-       solid= recast_newHeightfield();
+       solid = recast_newHeightfield();
 
-       if(!recast_createHeightfield(solid, width, height, bmin, bmax, recastParams->cellsize, recastParams->cellheight)) {
+       if (!recast_createHeightfield(solid, width, height, bmin, bmax, recastParams->cellsize, recastParams->cellheight)) {
                recast_destroyHeightfield(solid);
-
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to create height field");
+               return false;
        }
 
        /* Allocate array that can hold triangle flags */
-       triflags= MEM_callocN(sizeof(unsigned char)*ntris, "buildNavMesh triflags");
+       triflags = MEM_callocN(sizeof(unsigned char) * ntris, "buildNavMesh triflags");
 
        /* Find triangles which are walkable based on their slope and rasterize them */
-       recast_markWalkableTriangles(RAD2DEG(recastParams->agentmaxslope), verts, nverts, tris, ntris, triflags);
-       recast_rasterizeTriangles(verts, nverts, tris, triflags, ntris, solid);
+       recast_markWalkableTriangles(RAD2DEGF(recastParams->agentmaxslope), verts, nverts, tris, ntris, triflags);
+       recast_rasterizeTriangles(verts, nverts, tris, triflags, ntris, solid, 1);
        MEM_freeN(triflags);
 
        /* ** Step 3: Filter walkables surfaces ** */
@@ -219,12 +227,13 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 
        /* ** Step 4: Partition walkable surface to simple regions ** */
 
-       chf= recast_newCompactHeightfield();
-       if(!recast_buildCompactHeightfield(walkableHeight, walkableClimb, solid, chf)) {
+       chf = recast_newCompactHeightfield();
+       if (!recast_buildCompactHeightfield(walkableHeight, walkableClimb, solid, chf)) {
                recast_destroyHeightfield(solid);
                recast_destroyCompactHeightfield(chf);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to create compact height field");
+               return false;
        }
 
        recast_destroyHeightfield(solid);
@@ -233,73 +242,101 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
        if (!recast_erodeWalkableArea(walkableRadius, chf)) {
                recast_destroyCompactHeightfield(chf);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to erode walkable area");
+               return false;
        }
 
-       /* Prepare for region partitioning, by calculating distance field along the walkable surface */
-       if(!recast_buildDistanceField(chf)) {
-               recast_destroyCompactHeightfield(chf);
+       if (recastParams->partitioning == RC_PARTITION_WATERSHED) {
+               /* Prepare for region partitioning, by calculating distance field along the walkable surface */
+               if (!recast_buildDistanceField(chf)) {
+                       recast_destroyCompactHeightfield(chf);
 
-               return 0;
-       }
+                       BKE_report(reports, RPT_ERROR, "Failed to build distance field");
+                       return false;
+               }
 
-       /* Partition the walkable surface into simple regions without holes */
-       if(!recast_buildRegions(chf, 0, minRegionArea, mergeRegionArea)) {
-               recast_destroyCompactHeightfield(chf);
+               /* Partition the walkable surface into simple regions without holes */
+               if (!recast_buildRegions(chf, 0, minRegionArea, mergeRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
+
+                       BKE_report(reports, RPT_ERROR, "Failed to build watershed regions");
+                       return false;
+               }
+       }
+       else if (recastParams->partitioning == RC_PARTITION_MONOTONE) {
+               /* Partition the walkable surface into simple regions without holes */
+               /* Monotone partitioning does not need distancefield. */
+               if (!recast_buildRegionsMonotone(chf, 0, minRegionArea, mergeRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
+
+                       BKE_report(reports, RPT_ERROR, "Failed to build monotone regions");
+                       return false;
+               }
+       }
+       else { /* RC_PARTITION_LAYERS */
+               /* Partition the walkable surface into simple regions without holes */
+               if (!recast_buildLayerRegions(chf, 0, minRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
 
-               return 0;
+                       BKE_report(reports, RPT_ERROR, "Failed to build layer regions");
+                       return false;
+               }
        }
 
        /* ** Step 5: Trace and simplify region contours ** */
        /* Create contours */
-       cset= recast_newContourSet();
+       cset = recast_newContourSet();
 
-       if(!recast_buildContours(chf, recastParams->edgemaxerror, maxEdgeLen, cset)) {
+       if (!recast_buildContours(chf, recastParams->edgemaxerror, maxEdgeLen, cset, RECAST_CONTOUR_TESS_WALL_EDGES)) {
                recast_destroyCompactHeightfield(chf);
                recast_destroyContourSet(cset);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build contours");
+               return false;
        }
 
        /* ** Step 6: Build polygons mesh from contours ** */
-       *pmesh= recast_newPolyMesh();
-       if(!recast_buildPolyMesh(cset, recastParams->vertsperpoly, *pmesh)) {
+       *pmesh = recast_newPolyMesh();
+       if (!recast_buildPolyMesh(cset, recastParams->vertsperpoly, *pmesh)) {
                recast_destroyCompactHeightfield(chf);
                recast_destroyContourSet(cset);
                recast_destroyPolyMesh(*pmesh);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build poly mesh");
+               return false;
        }
 
 
        /* ** Step 7: Create detail mesh which allows to access approximate height on each polygon ** */
 
-       *dmesh= recast_newPolyMeshDetail();
-       if(!recast_buildPolyMeshDetail(*pmesh, chf, detailSampleDist, detailSampleMaxError, *dmesh)) {
+       *dmesh = recast_newPolyMeshDetail();
+       if (!recast_buildPolyMeshDetail(*pmesh, chf, detailSampleDist, detailSampleMaxError, *dmesh)) {
                recast_destroyCompactHeightfield(chf);
                recast_destroyContourSet(cset);
                recast_destroyPolyMesh(*pmesh);
                recast_destroyPolyMeshDetail(*dmesh);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build poly mesh detail");
+               return false;
        }
 
        recast_destroyCompactHeightfield(chf);
        recast_destroyContourSet(cset);
 
-       return 1;
+       return true;
 }
 
-static Object* createRepresentation(bContext *C, struct recast_polyMesh *pmesh, struct recast_polyMeshDetail *dmesh, Base* base)
+static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh, struct recast_polyMeshDetail *dmesh,
+                                  Base *base, unsigned int lay)
 {
        float co[3], rot[3];
        BMEditMesh *em;
-       int i,j, k;
-       unsigned shortv;
+       int i, j, k;
+       unsigned short *v;
        int face[3];
-       Scene *scene= CTX_data_scene(C);
-       Objectobedit;
-       int createob= base==NULL;
+       Scene *scene = CTX_data_scene(C);
+       Object *obedit;
+       int createob = base == NULL;
        int nverts, nmeshes, nvp;
        unsigned short *verts, *polys;
        unsigned int *meshes;
@@ -309,112 +346,112 @@ static Object* createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
        zero_v3(co);
        zero_v3(rot);
 
-       if(createob) {
+       if (createob) {
                /* create new object */
-               obedit= ED_object_add_type(C, OB_MESH, co, rot, FALSE, 1);
+               obedit = ED_object_add_type(C, OB_MESH, "Navmesh", co, rot, false, lay);
        }
        else {
-               obedit= base->object;
-               scene_select_base(scene, base);
+               obedit = base->object;
+               BKE_scene_base_deselect_all(scene);
+               BKE_scene_base_select(scene, base);
                copy_v3_v3(obedit->loc, co);
                copy_v3_v3(obedit->rot, rot);
        }
 
-       ED_object_enter_editmode(C, EM_DO_UNDO|EM_IGNORE_LAYER);
-       em= (((Mesh *)obedit->data))->edit_btmesh;
+       ED_object_editmode_enter(C, EM_DO_UNDO | EM_IGNORE_LAYER);
+       em = BKE_editmesh_from_object(obedit);
 
-       if(!createob) {
+       if (!createob) {
                /* clear */
-               EDBM_ClearMesh(em);
+               EDBM_mesh_clear(em);
        }
 
        /* create verts for polygon mesh */
-       verts= recast_polyMeshGetVerts(pmesh, &nverts);
+       verts = recast_polyMeshGetVerts(pmesh, &nverts);
        recast_polyMeshGetBoundbox(pmesh, bmin, NULL);
        recast_polyMeshGetCell(pmesh, &cs, &ch);
 
-       for(i= 0; i<nverts; i++) {
-               v= &verts[3*i];
-               co[0]= bmin[0] + v[0]*cs;
-               co[1]= bmin[1] + v[1]*ch;
-               co[2]= bmin[2] + v[2]*cs;
+       for (i = 0; i < nverts; i++) {
+               v = &verts[3 * i];
+               co[0] = bmin[0] + v[0] * cs;
+               co[1] = bmin[1] + v[1] * ch;
+               co[2] = bmin[2] + v[2] * cs;
                SWAP(float, co[1], co[2]);
-               BM_vert_create(em->bm, co, NULL);
+               BM_vert_create(em->bm, co, NULL, BM_CREATE_NOP);
        }
 
        /* create custom data layer to save polygon idx */
        CustomData_add_layer_named(&em->bm->pdata, CD_RECAST, CD_CALLOC, NULL, 0, "createRepresentation recastData");
+       CustomData_bmesh_init_pool(&em->bm->pdata, 0, BM_FACE);
        
        /* create verts and faces for detailed mesh */
-       meshes= recast_polyMeshDetailGetMeshes(dmesh, &nmeshes);
-       polys= recast_polyMeshGetPolys(pmesh, NULL, &nvp);
-       dverts= recast_polyMeshDetailGetVerts(dmesh, NULL);
-       tris= recast_polyMeshDetailGetTris(dmesh, NULL);
-
-       for(i= 0; i<nmeshes; i++) {
-               int uniquevbase= em->bm->totvert;
-               unsigned int vbase= meshes[4*i+0];
-               unsigned short ndv= meshes[4*i+1];
-               unsigned short tribase= meshes[4*i+2];
-               unsigned short trinum= meshes[4*i+3];
-               const unsigned short* p= &polys[i*nvp*2];
-               int nv= 0;
-
-               for(j= 0; j < nvp; ++j) {
-                       if(p[j]==0xffff) break;
+       meshes = recast_polyMeshDetailGetMeshes(dmesh, &nmeshes);
+       polys = recast_polyMeshGetPolys(pmesh, NULL, &nvp);
+       dverts = recast_polyMeshDetailGetVerts(dmesh, NULL);
+       tris = recast_polyMeshDetailGetTris(dmesh, NULL);
+
+       for (i = 0; i < nmeshes; i++) {
+               int uniquevbase = em->bm->totvert;
+               unsigned int vbase = meshes[4 * i + 0];
+               unsigned short ndv = meshes[4 * i + 1];
+               unsigned short tribase = meshes[4 * i + 2];
+               unsigned short trinum = meshes[4 * i + 3];
+               const unsigned short *p = &polys[i * nvp * 2];
+               int nv = 0;
+
+               for (j = 0; j < nvp; ++j) {
+                       if (p[j] == 0xffff) break;
                        nv++;
                }
 
                /* create unique verts  */
-               for(j= nv; j<ndv; j++) {
-                       copy_v3_v3(co, &dverts[3*(vbase + j)]);
+               for (j = nv; j < ndv; j++) {
+                       copy_v3_v3(co, &dverts[3 * (vbase + j)]);
                        SWAP(float, co[1], co[2]);
-                       BM_vert_create(em->bm, co, NULL);
+                       BM_vert_create(em->bm, co, NULL, BM_CREATE_NOP);
                }
 
-               EDBM_init_index_arrays(em, 1, 0, 0);
+               /* need to rebuild entirely because array size changes */
+               BM_mesh_elem_table_init(em->bm, BM_VERT);
 
                /* create faces */
-               for(j= 0; j<trinum; j++) {
-                       unsigned char* tri= &tris[4*(tribase+j)];
-                       BMFacenewFace;
-                       intpolygonIdx;
-
-                       for(k= 0; k<3; k++) {
-                               if(tri[k]<nv)
-                                       face[k] = p[tri[k]]; /* shared vertex */
+               for (j = 0; j < trinum; j++) {
+                       unsigned char *tri = &tris[4 * (tribase + j)];
+                       BMFace *newFace;
+                       int *polygonIdx;
+
+                       for (k = 0; k < 3; k++) {
+                               if (tri[k] < nv)
+                                       face[k] = p[tri[k]];  /* shared vertex */
                                else
-                                       face[k] = uniquevbase+tri[k]-nv; /* unique vertex */
+                                       face[k] = uniquevbase + tri[k] - nv;  /* unique vertex */
                        }
-                       newFace= BM_face_create_quad_tri(em->bm,
-                                                     EDBM_get_vert_for_index(em, face[0]),
-                                                     EDBM_get_vert_for_index(em, face[2]),
-                                                     EDBM_get_vert_for_index(em, face[1]), NULL,
-                                                     NULL, FALSE);
+                       newFace = BM_face_create_quad_tri(em->bm,
+                                                         BM_vert_at_index(em->bm, face[0]),
+                                                         BM_vert_at_index(em->bm, face[2]),
+                                                         BM_vert_at_index(em->bm, face[1]), NULL,
+                                                         NULL, BM_CREATE_NOP);
 
                        /* set navigation polygon idx to the custom layer */
-                       polygonIdx= (int*)CustomData_bmesh_get(&em->bm->pdata, newFace->head.data, CD_RECAST);
-                       *polygonIdx= i+1; /* add 1 to avoid zero idx */
+                       polygonIdx = (int *)CustomData_bmesh_get(&em->bm->pdata, newFace->head.data, CD_RECAST);
+                       *polygonIdx = i + 1; /* add 1 to avoid zero idx */
                }
-               
-               EDBM_free_index_arrays(em);
        }
 
        recast_destroyPolyMesh(pmesh);
        recast_destroyPolyMeshDetail(dmesh);
 
-       DAG_id_tag_update((ID*)obedit->data, OB_RECALC_DATA);
-       WM_event_add_notifier(C, NC_GEOM|ND_DATA, obedit->data);
+       DAG_id_tag_update((ID *)obedit->data, OB_RECALC_DATA);
+       WM_event_add_notifier(C, NC_GEOM | ND_DATA, obedit->data);
 
 
-       ED_object_exit_editmode(C, EM_FREEDATA); 
-       WM_event_add_notifier(C, NC_OBJECT|ND_DRAW, obedit);
+       ED_object_editmode_exit(C, EM_FREEDATA); 
+       WM_event_add_notifier(C, NC_OBJECT | ND_DRAW, obedit);
 
-       if(createob) {
-               obedit->gameflag&= ~OB_COLLISION;
-               obedit->gameflag|= OB_NAVMESH;
-               obedit->body_type= OB_BODY_TYPE_NAVMESH;
-               rename_id((ID *)obedit, "Navmesh");
+       if (createob) {
+               obedit->gameflag &= ~OB_COLLISION;
+               obedit->gameflag |= OB_NAVMESH;
+               obedit->body_type = OB_BODY_TYPE_NAVMESH;
        }
 
        BKE_mesh_ensure_navmesh(obedit->data);
@@ -422,43 +459,47 @@ static Object* createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
        return obedit;
 }
 
-static int create_navmesh_exec(bContext *C, wmOperator *op)
+static int navmesh_create_exec(bContext *C, wmOperator *op)
 {
-       Scene* scene= CTX_data_scene(C);
-       LinkNode* obs= NULL;
-       Base* navmeshBase= NULL;
+       Scene *scene = CTX_data_scene(C);
+       LinkNode *obs = NULL;
+       Base *navmeshBase = NULL;
 
-       CTX_DATA_BEGIN(C, Base*, base, selected_editable_bases) {
+       CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
+       {
                if (base->object->type == OB_MESH) {
-                       if (base->object->body_type==OB_BODY_TYPE_NAVMESH) {
+                       if (base->object->body_type == OB_BODY_TYPE_NAVMESH) {
                                if (!navmeshBase || base == scene->basact) {
-                                       navmeshBase= base;
+                                       navmeshBase = base;
                                }
                        }
                        else {
-                               BLI_linklist_append(&obs, (void*)base->object);
+                               BLI_linklist_prepend(&obs, base->object);
                        }
                }
        }
        CTX_DATA_END;
 
        if (obs) {
-               struct recast_polyMesh *pmesh= NULL;
-               struct recast_polyMeshDetail *dmesh= NULL;
+               struct recast_polyMesh *pmesh = NULL;
+               struct recast_polyMeshDetail *dmesh = NULL;
+               bool ok;
+               unsigned int lay = 0;
 
-               int nverts= 0, ntris= 0;
-               int *tris= 0;
-               float *verts= NULL;
+               int nverts = 0, ntris = 0;
+               int *tris = NULL;
+               float *verts = NULL;
 
-               createVertsTrisData(C, obs, &nverts, &verts, &ntris, &tris);
+               createVertsTrisData(C, obs, &nverts, &verts, &ntris, &tris, &lay);
                BLI_linklist_free(obs, NULL);
-               buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh);
-               createRepresentation(C, pmesh, dmesh, navmeshBase);
+               if ((ok = buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh, op->reports))) {
+                       createRepresentation(C, pmesh, dmesh, navmeshBase, lay);
+               }
 
                MEM_freeN(verts);
                MEM_freeN(tris);
 
-               return OPERATOR_FINISHED;
+               return ok ? OPERATOR_FINISHED : OPERATOR_CANCELLED;
        }
        else {
                BKE_report(op->reports, RPT_ERROR, "No mesh objects found");
@@ -470,38 +511,38 @@ static int create_navmesh_exec(bContext *C, wmOperator *op)
 void MESH_OT_navmesh_make(wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name= "Create navigation mesh";
-       ot->description= "Create navigation mesh for selected objects";
-       ot->idname= "MESH_OT_navmesh_make";
+       ot->name = "Create Navigation Mesh";
+       ot->description = "Create navigation mesh for selected objects";
+       ot->idname = "MESH_OT_navmesh_make";
 
        /* api callbacks */
-       ot->exec= create_navmesh_exec;
+       ot->exec = navmesh_create_exec;
 
        /* flags */
-       ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
+       ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
 static int navmesh_face_copy_exec(bContext *C, wmOperator *op)
 {
-       Object *obedit= CTX_data_edit_object(C);
-       BMEditMesh *em= ((Mesh *)obedit->data)->edit_btmesh;
+       Object *obedit = CTX_data_edit_object(C);
+       BMEditMesh *em = BKE_editmesh_from_object(obedit);
 
        /* do work here */
-       BMFace *efa_act= BM_active_face_get(em->bm, FALSE);
+       BMFace *efa_act = BM_mesh_active_face_get(em->bm, false, false);
 
-       if(efa_act) {
-               if(CustomData_has_layer(&em->bm->pdata, CD_RECAST)) {
+       if (efa_act) {
+               if (CustomData_has_layer(&em->bm->pdata, CD_RECAST)) {
                        BMFace *efa;
                        BMIter iter;
-                       int targetPolyIdx= *(int*)CustomData_bmesh_get(&em->bm->pdata, efa_act->head.data, CD_RECAST);
-                       targetPolyIdx= targetPolyIdx>=0? targetPolyIdx : -targetPolyIdx;
+                       int targetPolyIdx = *(int *)CustomData_bmesh_get(&em->bm->pdata, efa_act->head.data, CD_RECAST);
+                       targetPolyIdx = targetPolyIdx >= 0 ? targetPolyIdx : -targetPolyIdx;
 
-                       if(targetPolyIdx > 0) {
+                       if (targetPolyIdx > 0) {
                                /* set target poly idx to other selected faces */
-                               BM_ITER(efa, &iter, em->bm, BM_FACES_OF_MESH, NULL) {
-                                       if(BM_elem_flag_test(efa, BM_ELEM_SELECT) && efa != efa_act)  {
-                                               int* recastDataBlock= (int*)CustomData_bmesh_get(&em->bm->pdata, efa->head.data, CD_RECAST);
-                                               *recastDataBlock= targetPolyIdx;
+                               BM_ITER_MESH (efa, &iter, em->bm, BM_FACES_OF_MESH) {
+                                       if (BM_elem_flag_test(efa, BM_ELEM_SELECT) && efa != efa_act) {
+                                               int *recastDataBlock = (int *)CustomData_bmesh_get(&em->bm->pdata, efa->head.data, CD_RECAST);
+                                               *recastDataBlock = targetPolyIdx;
                                        }
                                }
                        }
@@ -511,8 +552,8 @@ static int navmesh_face_copy_exec(bContext *C, wmOperator *op)
                }
        }
 
-       DAG_id_tag_update((ID*)obedit->data, OB_RECALC_DATA);
-       WM_event_add_notifier(C, NC_GEOM|ND_DATA, obedit->data);
+       DAG_id_tag_update((ID *)obedit->data, OB_RECALC_DATA);
+       WM_event_add_notifier(C, NC_GEOM | ND_DATA, obedit->data);
 
        return OPERATOR_FINISHED;
 }
@@ -520,47 +561,47 @@ static int navmesh_face_copy_exec(bContext *C, wmOperator *op)
 void MESH_OT_navmesh_face_copy(struct wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name= "NavMesh Copy Face Index";
-       ot->description= "Copy the index from the active face";
-       ot->idname= "MESH_OT_navmesh_face_copy";
+       ot->name = "NavMesh Copy Face Index";
+       ot->description = "Copy the index from the active face";
+       ot->idname = "MESH_OT_navmesh_face_copy";
 
        /* api callbacks */
-       ot->poll= ED_operator_editmesh;
-       ot->exec= navmesh_face_copy_exec;
+       ot->poll = ED_operator_editmesh;
+       ot->exec = navmesh_face_copy_exec;
 
        /* flags */
-       ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
+       ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
-static int compare(const void * a, const void * b)
+static int compare(const void *a, const void *b)
 {
-       return ( *(int*)a - *(int*)b );
+       return (*(int *)a - *(int *)b);
 }
 
-static int findFreeNavPolyIndex(BMEditMeshem)
+static int findFreeNavPolyIndex(BMEditMesh *em)
 {
        /* construct vector of indices */
-       int numfaces= em->bm->totface;
-       int* indices= MEM_callocN(sizeof(int)*numfaces, "findFreeNavPolyIndex(indices)");
-       BMFaceef;
+       int numfaces = em->bm->totface;
+       int *indices = MEM_callocN(sizeof(int) * numfaces, "findFreeNavPolyIndex(indices)");
+       BMFace *ef;
        BMIter iter;
-       int i, idx= em->bm->totface-1, freeIdx= 1;
+       int i, idx = em->bm->totface - 1, freeIdx = 1;
 
        /*XXX this originally went last to first, but that isn't possible anymore*/
-       BM_ITER(ef, &iter, em->bm, BM_FACES_OF_MESH, NULL) {
-               int polyIdx= *(int*)CustomData_bmesh_get(&em->bm->pdata, ef->head.data, CD_RECAST);
-               indices[idx]= polyIdx;
+       BM_ITER_MESH (ef, &iter, em->bm, BM_FACES_OF_MESH) {
+               int polyIdx = *(int *)CustomData_bmesh_get(&em->bm->pdata, ef->head.data, CD_RECAST);
+               indices[idx] = polyIdx;
                idx--;
        }
 
        qsort(indices, numfaces, sizeof(int), compare);
 
        /* search first free index */
-       freeIdx= 1;
-       for(i= 0; i<numfaces; i++) {
-               if(indices[i]==freeIdx)
+       freeIdx = 1;
+       for (i = 0; i < numfaces; i++) {
+               if (indices[i] == freeIdx)
                        freeIdx++;
-               else if(indices[i]>freeIdx)
+               else if (indices[i] > freeIdx)
                        break;
        }
 
@@ -571,29 +612,29 @@ static int findFreeNavPolyIndex(BMEditMesh* em)
 
 static int navmesh_face_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
-       Object *obedit= CTX_data_edit_object(C);
-       BMEditMesh *em= ((Mesh *)obedit->data)->edit_btmesh;
+       Object *obedit = CTX_data_edit_object(C);
+       BMEditMesh *em = BKE_editmesh_from_object(obedit);
        BMFace *ef;
        BMIter iter;
        
-       if(CustomData_has_layer(&em->bm->pdata, CD_RECAST)) {
-               int targetPolyIdx= findFreeNavPolyIndex(em);
+       if (CustomData_has_layer(&em->bm->pdata, CD_RECAST)) {
+               int targetPolyIdx = findFreeNavPolyIndex(em);
 
-               if(targetPolyIdx>0) {
+               if (targetPolyIdx > 0) {
                        /* set target poly idx to selected faces */
                        /*XXX this originally went last to first, but that isn't possible anymore*/
                        
-                       BM_ITER(ef, &iter, em->bm, BM_FACES_OF_MESH, NULL) {
-                               if(BM_elem_flag_test(ef, BM_ELEM_SELECT)) {
-                                       int *recastDataBlock= (int*)CustomData_bmesh_get(&em->bm->pdata, ef->head.data, CD_RECAST);
-                                       *recastDataBlock= targetPolyIdx;
+                       BM_ITER_MESH (ef, &iter, em->bm, BM_FACES_OF_MESH) {
+                               if (BM_elem_flag_test(ef, BM_ELEM_SELECT)) {
+                                       int *recastDataBlock = (int *)CustomData_bmesh_get(&em->bm->pdata, ef->head.data, CD_RECAST);
+                                       *recastDataBlock = targetPolyIdx;
                                }
                        }
                }
        }
 
-       DAG_id_tag_update((ID*)obedit->data, OB_RECALC_DATA);
-       WM_event_add_notifier(C, NC_GEOM|ND_DATA, obedit->data);
+       DAG_id_tag_update((ID *)obedit->data, OB_RECALC_DATA);
+       WM_event_add_notifier(C, NC_GEOM | ND_DATA, obedit->data);
 
        return OPERATOR_FINISHED;
 }
@@ -601,48 +642,48 @@ static int navmesh_face_add_exec(bContext *C, wmOperator *UNUSED(op))
 void MESH_OT_navmesh_face_add(struct wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name= "NavMesh New Face Index";
-       ot->description= "Add a new index and assign it to selected faces";
-       ot->idname= "MESH_OT_navmesh_face_add";
+       ot->name = "NavMesh New Face Index";
+       ot->description = "Add a new index and assign it to selected faces";
+       ot->idname = "MESH_OT_navmesh_face_add";
 
        /* api callbacks */
-       ot->poll= ED_operator_editmesh;
-       ot->exec= navmesh_face_add_exec;
+       ot->poll = ED_operator_editmesh;
+       ot->exec = navmesh_face_add_exec;
 
        /* flags */
-       ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
+       ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
 static int navmesh_obmode_data_poll(bContext *C)
 {
        Object *ob = ED_object_active_context(C);
        if (ob && (ob->mode == OB_MODE_OBJECT) && (ob->type == OB_MESH)) {
-               Mesh *me= ob->data;
+               Mesh *me = ob->data;
                return CustomData_has_layer(&me->pdata, CD_RECAST);
        }
-       return FALSE;
+       return false;
 }
 
 static int navmesh_obmode_poll(bContext *C)
 {
        Object *ob = ED_object_active_context(C);
        if (ob && (ob->mode == OB_MODE_OBJECT) && (ob->type == OB_MESH)) {
-               return TRUE;
+               return true;
        }
-       return FALSE;
+       return false;
 }
 
 static int navmesh_reset_exec(bContext *C, wmOperator *UNUSED(op))
 {
        Object *ob = ED_object_active_context(C);
-       Mesh *me= ob->data;
+       Mesh *me = ob->data;
 
        CustomData_free_layers(&me->pdata, CD_RECAST, me->totpoly);
 
        BKE_mesh_ensure_navmesh(me);
 
        DAG_id_tag_update(&me->id, OB_RECALC_DATA);
-       WM_event_add_notifier(C, NC_GEOM|ND_DATA, &me->id);
+       WM_event_add_notifier(C, NC_GEOM | ND_DATA, &me->id);
 
        return OPERATOR_FINISHED;
 }
@@ -650,27 +691,27 @@ static int navmesh_reset_exec(bContext *C, wmOperator *UNUSED(op))
 void MESH_OT_navmesh_reset(struct wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name= "NavMesh Reset Index Values";
-       ot->description= "Assign a new index to every face";
-       ot->idname= "MESH_OT_navmesh_reset";
+       ot->name = "NavMesh Reset Index Values";
+       ot->description = "Assign a new index to every face";
+       ot->idname = "MESH_OT_navmesh_reset";
 
        /* api callbacks */
-       ot->poll= navmesh_obmode_poll;
-       ot->exec= navmesh_reset_exec;
+       ot->poll = navmesh_obmode_poll;
+       ot->exec = navmesh_reset_exec;
 
        /* flags */
-       ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
+       ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
 static int navmesh_clear_exec(bContext *C, wmOperator *UNUSED(op))
 {
        Object *ob = ED_object_active_context(C);
-       Mesh *me= ob->data;
+       Mesh *me = ob->data;
 
        CustomData_free_layers(&me->pdata, CD_RECAST, me->totpoly);
 
        DAG_id_tag_update(&me->id, OB_RECALC_DATA);
-       WM_event_add_notifier(C, NC_GEOM|ND_DATA, &me->id);
+       WM_event_add_notifier(C, NC_GEOM | ND_DATA, &me->id);
 
        return OPERATOR_FINISHED;
 }
@@ -678,14 +719,14 @@ static int navmesh_clear_exec(bContext *C, wmOperator *UNUSED(op))
 void MESH_OT_navmesh_clear(struct wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name= "NavMesh Clear Data";
-       ot->description= "Remove navmesh data from this mesh";
-       ot->idname= "MESH_OT_navmesh_clear";
+       ot->name = "NavMesh Clear Data";
+       ot->description = "Remove navmesh data from this mesh";
+       ot->idname = "MESH_OT_navmesh_clear";
 
        /* api callbacks */
-       ot->poll= navmesh_obmode_data_poll;
-       ot->exec= navmesh_clear_exec;
+       ot->poll = navmesh_obmode_data_poll;
+       ot->exec = navmesh_clear_exec;
 
        /* flags */
-       ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
+       ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }