fix for is_quad_convex_v3(), getting the dominant axis wasn't accurate enough in some cases and would make beauty fill fail.

now rotate the coords before calculation.

source/blender/blenlib/BLI_math_geom.h

@@ -262,6 +262,7 @@ MINLINE void madd_sh_shfl(float r[9], const float sh[3], const float f);
 float form_factor_hemi_poly(float p[3], float n[3],
                             float v1[3], float v2[3], float v3[3], float v4[3]);
 
+bool  axis_dominant_v3_to_m3(float r_mat[3][3], const float normal[3]);
 void  axis_dominant_v3(int *r_axis_a, int *r_axis_b, const float axis[3]);
 float axis_dominant_v3_max(int *r_axis_a, int *r_axis_b, const float axis[3])
 #ifdef __GNUC__

source/blender/blenlib/intern/math_geom.c

@@ -1966,7 +1966,48 @@ void plot_line_v2v2i(const int p1[2], const int p2[2], int (*callback)(int, int,
 	}
 }
 
-/****************************** Interpolation ********************************/
+/****************************** Axis Utils ********************************/
+
+/**
+ * \brief Normal to x,y matrix
+ *
+ * Creates a 3x3 matrix from a normal.
+ * This matrix can be applied to vectors so their 'z' axis runs along \a normal.
+ * In practice it means you can use x,y as 2d coords. \see
+ *
+ * \param r_mat The matrix to return.
+ * \param normal A unit length vector.
+ */
+bool axis_dominant_v3_to_m3(float r_mat[3][3], const float normal[3])
+{
+	float up[3] = {0.0f, 0.0f, 1.0f};
+	float axis[3];
+	float angle;
+
+	/* double check they are normalized */
+#ifdef DEBUG
+	float test;
+	BLI_assert(fabsf((test = len_squared_v3(normal)) - 1.0f) < 0.0001f || fabsf(test) < 0.0001f);
+#endif
+
+	cross_v3_v3v3(axis, normal, up);
+	angle = saacos(dot_v3v3(normal, up));
+
+	if (angle >= FLT_EPSILON) {
+		if (len_squared_v3(axis) < FLT_EPSILON) {
+			axis[0] = 0.0f;
+			axis[1] = 1.0f;
+			axis[2] = 0.0f;
+		}
+
+		axis_angle_to_mat3(r_mat, axis, angle);
+		return true;
+	}
+	else {
+		unit_m3(r_mat);
+		return false;
+	}
+}
 
 /* get the 2 dominant axis values, 0==X, 1==Y, 2==Z */
 void axis_dominant_v3(int *r_axis_a, int *r_axis_b, const float axis[3])
@@ -1992,6 +2033,9 @@ float axis_dominant_v3_max(int *r_axis_a, int *r_axis_b, const float axis[3])
 	else                           { *r_axis_a = 1; *r_axis_b = 2; return xn; }
 }
 
+
+/****************************** Interpolation ********************************/
+
 static float tri_signed_area(const float v1[3], const float v2[3], const float v3[3], const int i, const int j)
 {
 	return 0.5f * ((v1[i] - v2[i]) * (v2[j] - v3[j]) + (v1[j] - v2[j]) * (v3[i] - v2[i]));
@@ -3535,7 +3579,7 @@ float form_factor_hemi_poly(float p[3], float n[3], float v1[3], float v2[3], fl
 int is_quad_convex_v3(const float v1[3], const float v2[3], const float v3[3], const float v4[3])
 {
 	float nor[3], nor1[3], nor2[3], vec[4][2];
-	int axis_a, axis_b;
+	float mat[3][3];
 
 	/* define projection, do both trias apart, quad is undefined! */
 
@@ -3552,18 +3596,14 @@ int is_quad_convex_v3(const float v1[3], const float v2[3], const float v3[3], c
 	}
 
 	add_v3_v3v3(nor, nor1, nor2);
+	normalize_v3(nor);
 
-	axis_dominant_v3(&axis_a, &axis_b, nor);
+	axis_dominant_v3_to_m3(mat, nor);
 
-	vec[0][0] = v1[axis_a];
-	vec[0][1] = v1[axis_b];
-	vec[1][0] = v2[axis_a];
-	vec[1][1] = v2[axis_b];
-
-	vec[2][0] = v3[axis_a];
-	vec[2][1] = v3[axis_b];
-	vec[3][0] = v4[axis_a];
-	vec[3][1] = v4[axis_b];
+	mul_v2_m3v3(vec[0], mat, v1);
+	mul_v2_m3v3(vec[1], mat, v2);
+	mul_v2_m3v3(vec[2], mat, v3);
+	mul_v2_m3v3(vec[3], mat, v4);
 
 	/* linetests, the 2 diagonals have to instersect to be convex */
 	return (isect_line_line_v2(vec[0], vec[2], vec[1], vec[3]) > 0) ? TRUE : FALSE;

source/blender/bmesh/intern/bmesh_polygon.c

@@ -297,35 +297,6 @@ static void scale_edge_v3f(float v1[3], float v2[3], const float fac)
 	add_v3_v3v3(v2, v2, mid);
 }
 
-/**
- * \brief POLY NORMAL TO MATRIX
- *
- * Creates a 3x3 matrix from a normal.
- */
-static bool poly_normal_to_xy_mat3(float r_mat[3][3], const float normal[3])
-{
-	float up[3] = {0.0f, 0.0f, 1.0f}, axis[3];
-	float angle;
-
-	cross_v3_v3v3(axis, normal, up);
-	angle = saacos(dot_v3v3(normal, up));
-
-	if (angle >= FLT_EPSILON) {
-		if (len_squared_v3(axis) < FLT_EPSILON) {
-			axis[0] = 0.0f;
-			axis[1] = 1.0f;
-			axis[2] = 0.0f;
-		}
-
-		axis_angle_to_mat3(r_mat, axis, angle);
-		return true;
-	}
-	else {
-		unit_m3(r_mat);
-		return false;
-	}
-}
-
 /**
  * \brief POLY ROTATE PLANE
  *
@@ -336,7 +307,7 @@ void poly_rotate_plane(const float normal[3], float (*verts)[3], const int nvert
 {
 	float mat[3][3];
 
-	if (poly_normal_to_xy_mat3(mat, normal)) {
+	if (axis_dominant_v3_to_m3(mat, normal)) {
 		int i;
 		for (i = 0; i < nverts; i++) {
 			mul_m3_v3(mat, verts[i]);
@@ -828,7 +799,7 @@ void BM_face_triangulate(BMesh *bm, BMFace *f,
 	float *abscoss = BLI_array_alloca(abscoss, f_len_orig);
 	float mat[3][3];
 
-	poly_normal_to_xy_mat3(mat, f->no);
+	axis_dominant_v3_to_m3(mat, f->no);
 
 	/* copy vertex coordinates to vertspace area */
 	i = 0;