Fix navmesh creation w/ multiple objects
[blender-staging.git] / source / blender / editors / mesh / mesh_navmesh.c
index 71aaacb7e492529ebc4113380e9fd0097f26eb09..740d18951dcb7cceb8497f7ba998404bf7be237c 100644 (file)
  * ***** END GPL LICENSE BLOCK *****
  */
 
-#include <math.h>
+/** \file blender/editors/mesh/mesh_navmesh.c
+ *  \ingroup edmesh
+ */
 
 #include "MEM_guardedalloc.h"
 
 #include "DNA_scene_types.h"
 #include "DNA_object_types.h"
 #include "DNA_mesh_types.h"
-#include "DNA_meshdata_types.h"
-#include "DNA_modifier_types.h"
-#include "DNA_ID.h"
 
 #include "BLI_listbase.h"
-#include "BLI_utildefines.h"
 #include "BLI_math_vector.h"
 #include "BLI_linklist.h"
 
 #include "BKE_library.h"
 #include "BKE_depsgraph.h"
 #include "BKE_context.h"
-#include "BKE_main.h"
 #include "BKE_mesh.h"
-#include "BKE_modifier.h"
 #include "BKE_scene.h"
 #include "BKE_DerivedMesh.h"
-#include "BKE_cdderivedmesh.h"
 #include "BKE_report.h"
-#include "BKE_tessmesh.h"
+#include "BKE_editmesh.h"
 
 #include "ED_object.h"
 #include "ED_mesh.h"
 #include "ED_screen.h"
 
-#include "RNA_access.h"
-
 #include "WM_api.h"
 #include "WM_types.h"
 
-#include "mesh_intern.h"
 #include "recast-capi.h"
 
-static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float **verts_r, int *ntris_r, int **tris_r)
+#include "mesh_intern.h"  /* own include */
+
+
+static void createVertsTrisData(bContext *C, LinkNode *obs,
+                                int *nverts_r, float **verts_r, int *ntris_r, int **tris_r, unsigned int *r_lay)
 {
        MVert *mvert;
        int nfaces = 0, *tri, i, curnverts, basenverts, curnfaces;
@@ -76,7 +72,7 @@ static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float
        LinkNode *oblink, *dmlink;
        DerivedMesh *dm;
        Scene *scene = CTX_data_scene(C);
-       LinkNode *dms = NULL;
+       LinkNodePair dms_pair = {NULL,NULL};
 
        int nverts, ntris, *tris;
        float *verts;
@@ -88,7 +84,8 @@ static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float
        for (oblink = obs; oblink; oblink = oblink->next) {
                ob = (Object *) oblink->link;
                dm = mesh_create_derived_no_virtual(scene, ob, NULL, CD_MASK_MESH);
-               BLI_linklist_append(&dms, (void *)dm);
+               DM_ensure_tessface(dm);
+               BLI_linklist_append(&dms_pair, dm);
 
                nverts += dm->getNumVerts(dm);
                nfaces = dm->getNumTessFaces(dm);
@@ -101,7 +98,10 @@ static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float
                        if (mf->v4)
                                ntris += 1;
                }
+
+               *r_lay |= ob->lay;
        }
+       LinkNode *dms = dms_pair.list;
 
        /* create data */
        verts = MEM_mallocN(sizeof(float) * 3 * nverts, "createVertsTrisData verts");
@@ -167,8 +167,9 @@ static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float
        *tris_r = tris;
 }
 
-static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
-                        struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh)
+static bool buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
+                         struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh,
+                         ReportList *reports)
 {
        float bmin[3], bmax[3];
        struct recast_heightfield *solid;
@@ -195,22 +196,28 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
        /* Set the area where the navigation will be build. */
        recast_calcGridSize(bmin, bmax, recastParams->cellsize, &width, &height);
 
+       /* zero dimensions cause zero alloc later on [#33758] */
+       if (width <= 0 || height <= 0) {
+               BKE_report(reports, RPT_ERROR, "Object has a width or height of zero");
+               return false;
+       }
+
        /* ** Step 2: Rasterize input polygon soup ** */
        /* Allocate voxel heightfield where we rasterize our input data to */
        solid = recast_newHeightfield();
 
        if (!recast_createHeightfield(solid, width, height, bmin, bmax, recastParams->cellsize, recastParams->cellheight)) {
                recast_destroyHeightfield(solid);
-
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to create height field");
+               return false;
        }
 
        /* Allocate array that can hold triangle flags */
        triflags = MEM_callocN(sizeof(unsigned char) * ntris, "buildNavMesh triflags");
 
        /* Find triangles which are walkable based on their slope and rasterize them */
-       recast_markWalkableTriangles(RAD2DEG(recastParams->agentmaxslope), verts, nverts, tris, ntris, triflags);
-       recast_rasterizeTriangles(verts, nverts, tris, triflags, ntris, solid);
+       recast_markWalkableTriangles(RAD2DEGF(recastParams->agentmaxslope), verts, nverts, tris, ntris, triflags);
+       recast_rasterizeTriangles(verts, nverts, tris, triflags, ntris, solid, 1);
        MEM_freeN(triflags);
 
        /* ** Step 3: Filter walkables surfaces ** */
@@ -225,7 +232,8 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
                recast_destroyHeightfield(solid);
                recast_destroyCompactHeightfield(chf);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to create compact height field");
+               return false;
        }
 
        recast_destroyHeightfield(solid);
@@ -234,32 +242,57 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
        if (!recast_erodeWalkableArea(walkableRadius, chf)) {
                recast_destroyCompactHeightfield(chf);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to erode walkable area");
+               return false;
        }
 
-       /* Prepare for region partitioning, by calculating distance field along the walkable surface */
-       if (!recast_buildDistanceField(chf)) {
-               recast_destroyCompactHeightfield(chf);
+       if (recastParams->partitioning == RC_PARTITION_WATERSHED) {
+               /* Prepare for region partitioning, by calculating distance field along the walkable surface */
+               if (!recast_buildDistanceField(chf)) {
+                       recast_destroyCompactHeightfield(chf);
 
-               return 0;
-       }
+                       BKE_report(reports, RPT_ERROR, "Failed to build distance field");
+                       return false;
+               }
 
-       /* Partition the walkable surface into simple regions without holes */
-       if (!recast_buildRegions(chf, 0, minRegionArea, mergeRegionArea)) {
-               recast_destroyCompactHeightfield(chf);
+               /* Partition the walkable surface into simple regions without holes */
+               if (!recast_buildRegions(chf, 0, minRegionArea, mergeRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
+
+                       BKE_report(reports, RPT_ERROR, "Failed to build watershed regions");
+                       return false;
+               }
+       }
+       else if (recastParams->partitioning == RC_PARTITION_MONOTONE) {
+               /* Partition the walkable surface into simple regions without holes */
+               /* Monotone partitioning does not need distancefield. */
+               if (!recast_buildRegionsMonotone(chf, 0, minRegionArea, mergeRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
+
+                       BKE_report(reports, RPT_ERROR, "Failed to build monotone regions");
+                       return false;
+               }
+       }
+       else { /* RC_PARTITION_LAYERS */
+               /* Partition the walkable surface into simple regions without holes */
+               if (!recast_buildLayerRegions(chf, 0, minRegionArea)) {
+                       recast_destroyCompactHeightfield(chf);
 
-               return 0;
+                       BKE_report(reports, RPT_ERROR, "Failed to build layer regions");
+                       return false;
+               }
        }
 
        /* ** Step 5: Trace and simplify region contours ** */
        /* Create contours */
        cset = recast_newContourSet();
 
-       if (!recast_buildContours(chf, recastParams->edgemaxerror, maxEdgeLen, cset)) {
+       if (!recast_buildContours(chf, recastParams->edgemaxerror, maxEdgeLen, cset, RECAST_CONTOUR_TESS_WALL_EDGES)) {
                recast_destroyCompactHeightfield(chf);
                recast_destroyContourSet(cset);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build contours");
+               return false;
        }
 
        /* ** Step 6: Build polygons mesh from contours ** */
@@ -269,7 +302,8 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
                recast_destroyContourSet(cset);
                recast_destroyPolyMesh(*pmesh);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build poly mesh");
+               return false;
        }
 
 
@@ -282,16 +316,18 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
                recast_destroyPolyMesh(*pmesh);
                recast_destroyPolyMeshDetail(*dmesh);
 
-               return 0;
+               BKE_report(reports, RPT_ERROR, "Failed to build poly mesh detail");
+               return false;
        }
 
        recast_destroyCompactHeightfield(chf);
        recast_destroyContourSet(cset);
 
-       return 1;
+       return true;
 }
 
-static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh, struct recast_polyMeshDetail *dmesh, Base *base)
+static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh, struct recast_polyMeshDetail *dmesh,
+                                  Base *base, unsigned int lay)
 {
        float co[3], rot[3];
        BMEditMesh *em;
@@ -312,17 +348,18 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
 
        if (createob) {
                /* create new object */
-               obedit = ED_object_add_type(C, OB_MESH, co, rot, FALSE, 1);
+               obedit = ED_object_add_type(C, OB_MESH, "Navmesh", co, rot, false, lay);
        }
        else {
                obedit = base->object;
-               scene_select_base(scene, base);
+               BKE_scene_base_deselect_all(scene);
+               BKE_scene_base_select(scene, base);
                copy_v3_v3(obedit->loc, co);
                copy_v3_v3(obedit->rot, rot);
        }
 
-       ED_object_enter_editmode(C, EM_DO_UNDO | EM_IGNORE_LAYER);
-       em = BMEdit_FromObject(obedit);
+       ED_object_editmode_enter(C, EM_DO_UNDO | EM_IGNORE_LAYER);
+       em = BKE_editmesh_from_object(obedit);
 
        if (!createob) {
                /* clear */
@@ -340,7 +377,7 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
                co[1] = bmin[1] + v[1] * ch;
                co[2] = bmin[2] + v[2] * cs;
                SWAP(float, co[1], co[2]);
-               BM_vert_create(em->bm, co, NULL);
+               BM_vert_create(em->bm, co, NULL, BM_CREATE_NOP);
        }
 
        /* create custom data layer to save polygon idx */
@@ -371,10 +408,11 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
                for (j = nv; j < ndv; j++) {
                        copy_v3_v3(co, &dverts[3 * (vbase + j)]);
                        SWAP(float, co[1], co[2]);
-                       BM_vert_create(em->bm, co, NULL);
+                       BM_vert_create(em->bm, co, NULL, BM_CREATE_NOP);
                }
 
-               EDBM_index_arrays_init(em, 1, 0, 0);
+               /* need to rebuild entirely because array size changes */
+               BM_mesh_elem_table_init(em->bm, BM_VERT);
 
                /* create faces */
                for (j = 0; j < trinum; j++) {
@@ -389,17 +427,15 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
                                        face[k] = uniquevbase + tri[k] - nv;  /* unique vertex */
                        }
                        newFace = BM_face_create_quad_tri(em->bm,
-                                                         EDBM_vert_at_index(em, face[0]),
-                                                         EDBM_vert_at_index(em, face[2]),
-                                                         EDBM_vert_at_index(em, face[1]), NULL,
-                                                         NULL, FALSE);
+                                                         BM_vert_at_index(em->bm, face[0]),
+                                                         BM_vert_at_index(em->bm, face[2]),
+                                                         BM_vert_at_index(em->bm, face[1]), NULL,
+                                                         NULL, BM_CREATE_NOP);
 
                        /* set navigation polygon idx to the custom layer */
                        polygonIdx = (int *)CustomData_bmesh_get(&em->bm->pdata, newFace->head.data, CD_RECAST);
                        *polygonIdx = i + 1; /* add 1 to avoid zero idx */
                }
-               
-               EDBM_index_arrays_free(em);
        }
 
        recast_destroyPolyMesh(pmesh);
@@ -409,14 +445,13 @@ static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh,
        WM_event_add_notifier(C, NC_GEOM | ND_DATA, obedit->data);
 
 
-       ED_object_exit_editmode(C, EM_FREEDATA); 
+       ED_object_editmode_exit(C, EM_FREEDATA); 
        WM_event_add_notifier(C, NC_OBJECT | ND_DRAW, obedit);
 
        if (createob) {
                obedit->gameflag &= ~OB_COLLISION;
                obedit->gameflag |= OB_NAVMESH;
                obedit->body_type = OB_BODY_TYPE_NAVMESH;
-               rename_id((ID *)obedit, "Navmesh");
        }
 
        BKE_mesh_ensure_navmesh(obedit->data);
@@ -430,7 +465,8 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
        LinkNode *obs = NULL;
        Base *navmeshBase = NULL;
 
-       CTX_DATA_BEGIN(C, Base*, base, selected_editable_bases) {
+       CTX_DATA_BEGIN (C, Base *, base, selected_editable_bases)
+       {
                if (base->object->type == OB_MESH) {
                        if (base->object->body_type == OB_BODY_TYPE_NAVMESH) {
                                if (!navmeshBase || base == scene->basact) {
@@ -438,7 +474,7 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
                                }
                        }
                        else {
-                               BLI_linklist_append(&obs, (void *)base->object);
+                               BLI_linklist_prepend(&obs, base->object);
                        }
                }
        }
@@ -447,20 +483,23 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
        if (obs) {
                struct recast_polyMesh *pmesh = NULL;
                struct recast_polyMeshDetail *dmesh = NULL;
+               bool ok;
+               unsigned int lay = 0;
 
                int nverts = 0, ntris = 0;
-               int *tris = 0;
+               int *tris = NULL;
                float *verts = NULL;
 
-               createVertsTrisData(C, obs, &nverts, &verts, &ntris, &tris);
+               createVertsTrisData(C, obs, &nverts, &verts, &ntris, &tris, &lay);
                BLI_linklist_free(obs, NULL);
-               buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh);
-               createRepresentation(C, pmesh, dmesh, navmeshBase);
+               if ((ok = buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh, op->reports))) {
+                       createRepresentation(C, pmesh, dmesh, navmeshBase, lay);
+               }
 
                MEM_freeN(verts);
                MEM_freeN(tris);
 
-               return OPERATOR_FINISHED;
+               return ok ? OPERATOR_FINISHED : OPERATOR_CANCELLED;
        }
        else {
                BKE_report(op->reports, RPT_ERROR, "No mesh objects found");
@@ -472,7 +511,7 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
 void MESH_OT_navmesh_make(wmOperatorType *ot)
 {
        /* identifiers */
-       ot->name = "Create navigation mesh";
+       ot->name = "Create Navigation Mesh";
        ot->description = "Create navigation mesh for selected objects";
        ot->idname = "MESH_OT_navmesh_make";
 
@@ -486,10 +525,10 @@ void MESH_OT_navmesh_make(wmOperatorType *ot)
 static int navmesh_face_copy_exec(bContext *C, wmOperator *op)
 {
        Object *obedit = CTX_data_edit_object(C);
-       BMEditMesh *em = BMEdit_FromObject(obedit);
+       BMEditMesh *em = BKE_editmesh_from_object(obedit);
 
        /* do work here */
-       BMFace *efa_act = BM_active_face_get(em->bm, FALSE);
+       BMFace *efa_act = BM_mesh_active_face_get(em->bm, false, false);
 
        if (efa_act) {
                if (CustomData_has_layer(&em->bm->pdata, CD_RECAST)) {
@@ -574,7 +613,7 @@ static int findFreeNavPolyIndex(BMEditMesh *em)
 static int navmesh_face_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
        Object *obedit = CTX_data_edit_object(C);
-       BMEditMesh *em = BMEdit_FromObject(obedit);
+       BMEditMesh *em = BKE_editmesh_from_object(obedit);
        BMFace *ef;
        BMIter iter;
        
@@ -622,16 +661,16 @@ static int navmesh_obmode_data_poll(bContext *C)
                Mesh *me = ob->data;
                return CustomData_has_layer(&me->pdata, CD_RECAST);
        }
-       return FALSE;
+       return false;
 }
 
 static int navmesh_obmode_poll(bContext *C)
 {
        Object *ob = ED_object_active_context(C);
        if (ob && (ob->mode == OB_MODE_OBJECT) && (ob->type == OB_MESH)) {
-               return TRUE;
+               return true;
        }
-       return FALSE;
+       return false;
 }
 
 static int navmesh_reset_exec(bContext *C, wmOperator *UNUSED(op))