author Campbell Barton Sat, 9 Feb 2013 08:16:13 +0000 (08:16 +0000) committer Campbell Barton Sat, 9 Feb 2013 08:16:13 +0000 (08:16 +0000)
Project the triangle pair into 2d coords before measuring.

before/after - http://www.graphicall.org/ftp/ideasman42/beauty_fill_fix.png

index 1007e05..563fccf 100644 (file)
@@ -75,7 +75,9 @@ void bmo_beautify_fill_exec(BMesh *bm, BMOperator *op)
stop = 1;

BM_ITER_MESH (e, &iter, bm, BM_EDGES_OF_MESH) {
-                       BMVert *v1, *v2, *v3, *v4;
+                       float v1_xy[2], v2_xy[2], v3_xy[2], v4_xy[2];
+                       float no[3];
+                       float axis_mat[3][3];

if (!BM_edge_is_manifold(e) || !BMO_elem_flag_test(bm, e, EDGE_MARK)) {
continue;
@@ -87,30 +89,44 @@ void bmo_beautify_fill_exec(BMesh *bm, BMOperator *op)
continue;
}

-                       v1 = e->l->prev->v;
-                       v2 = e->l->v;
-                       v4 = e->l->next->v;
-
-                       if (is_quad_convex_v3(v1->co, v2->co, v3->co, v4->co)) {
+                       {
+                               float *v1, *v2, *v3, *v4;
+                               float no_a[3], no_b[3];
+                               v1 = e->l->prev->v->co;
+                               v2 = e->l->v->co;
+                               v4 = e->l->next->v->co;
+
+                               normal_tri_v3(no_a, v1, v2, v3);
+                               normal_tri_v3(no_b, v1, v3, v4);
+                               normalize_v3(no);
+                               axis_dominant_v3_to_m3(axis_mat, no);
+                               mul_v2_m3v3(v1_xy, axis_mat, v1);
+                               mul_v2_m3v3(v2_xy, axis_mat, v2);
+                               mul_v2_m3v3(v3_xy, axis_mat, v3);
+                               mul_v2_m3v3(v4_xy, axis_mat, v4);
+                       }
+
+                       if (is_quad_convex_v2(v1_xy, v2_xy, v3_xy, v4_xy)) {
float len1, len2, len3, len4, len5, len6, opp1, opp2, fac1, fac2;
/* testing rule:
* the area divided by the total edge lengths
*/
-                               len1 = len_v3v3(v1->co, v2->co);
-                               len2 = len_v3v3(v2->co, v3->co);
-                               len3 = len_v3v3(v3->co, v4->co);
-                               len4 = len_v3v3(v4->co, v1->co);
-                               len5 = len_v3v3(v1->co, v3->co);
-                               len6 = len_v3v3(v2->co, v4->co);
+                               len1 = len_v2v2(v1_xy, v2_xy);
+                               len2 = len_v2v2(v2_xy, v3_xy);
+                               len3 = len_v2v2(v3_xy, v4_xy);
+                               len4 = len_v2v2(v4_xy, v1_xy);
+                               len5 = len_v2v2(v1_xy, v3_xy);
+                               len6 = len_v2v2(v2_xy, v4_xy);

-                               opp1 = area_tri_v3(v1->co, v2->co, v3->co);
-                               opp2 = area_tri_v3(v1->co, v3->co, v4->co);
+                               opp1 = area_tri_v2(v1_xy, v2_xy, v3_xy);
+                               opp2 = area_tri_v2(v1_xy, v3_xy, v4_xy);

fac1 = opp1 / (len1 + len2 + len5) + opp2 / (len3 + len4 + len5);

-                               opp1 = area_tri_v3(v2->co, v3->co, v4->co);
-                               opp2 = area_tri_v3(v2->co, v4->co, v1->co);
+                               opp1 = area_tri_v2(v2_xy, v3_xy, v4_xy);
+                               opp2 = area_tri_v2(v2_xy, v4_xy, v1_xy);

fac2 = opp1 / (len2 + len3 + len6) + opp2 / (len4 + len1 + len6);