Fix cross_poly_v2 returning a negated value, add tests

The result of cross_poly_v2 was flipped compared with cross_tri_v2 &
cross_poly_v3 (with the Z values zeroed).

Ensure cross_poly_v2/3, cross_tri_v2/3 return compatible results and
updating the doc-strings noting that a negative Z is for clock-wise
polygons.

source/blender/blenkernel/intern/mesh_mapping.cc

@@ -106,7 +106,7 @@ UvVertMap *BKE_mesh_uv_vert_map_create(const blender::OffsetIndices<int> faces,
       }
 
       if (use_winding) {
-        winding[a] = cross_poly_v2(tf_uv, uint(nverts)) > 0;
+        winding[a] = cross_poly_v2(tf_uv, uint(nverts)) < 0;
       }
     }
   }

source/blender/blenkernel/intern/pbvh_uv_islands.cc

@@ -1186,7 +1186,7 @@ bool UVBorder::is_ccw() const
   copy_v2_v2(poly[0], uv_vertex1->uv);
   copy_v2_v2(poly[1], uv_vertex2->uv);
   copy_v2_v2(poly[2], uv_vertex3->uv);
-  const bool ccw = cross_poly_v2(poly, 3) < 0.0;
+  const bool ccw = cross_poly_v2(poly, 3) > 0.0;
   return ccw;
 }
 

source/blender/blenlib/BLI_math_geom.h

@@ -57,13 +57,19 @@ float area_poly_signed_v2(const float verts[][2], unsigned int nr);
 float cotangent_tri_weight_v3(const float v1[3], const float v2[3], const float v3[3]);
 
 void cross_tri_v3(float n[3], const float v1[3], const float v2[3], const float v3[3]);
+/**
+ * Scalar cross product of a 2D triangle.
+ *
+ * - Equivalent to `area * 2`.
+ * - Useful for checking polygon winding (a negative value is clockwise).
+ */
 MINLINE float cross_tri_v2(const float v1[2], const float v2[2], const float v3[2]);
 void cross_poly_v3(float n[3], const float verts[][3], unsigned int nr);
 /**
- * Scalar cross product of a 2d polygon.
+ * Scalar cross product of a 2D polygon.
  *
- * - equivalent to `area * 2`
- * - useful for checking polygon winding (a positive value is clockwise).
+ * - Equivalent to `area * 2`.
+ * - Useful for checking polygon winding (a negative value is clockwise).
  */
 float cross_poly_v2(const float verts[][2], unsigned int nr);
 

source/blender/blenlib/intern/math_geom.cc

@@ -156,7 +156,7 @@ float cross_poly_v2(const float verts[][2], uint nr)
   co_curr = verts[0];
   cross = 0.0f;
   for (a = 0; a < nr; a++) {
-    cross += (co_curr[0] - co_prev[0]) * (co_curr[1] + co_prev[1]);
+    cross += (co_prev[0] - co_curr[0]) * (co_curr[1] + co_prev[1]);
     co_prev = co_curr;
     co_curr += 2;
   }

source/blender/blenlib/intern/polyfill_2d.c

@@ -799,17 +799,17 @@ static void polyfill_prepare(PolyFill *pf,
   pf->tris_num = 0;
 
   if (coords_sign == 0) {
-    coords_sign = (cross_poly_v2(coords, coords_num) >= 0.0f) ? 1 : -1;
+    coords_sign = (cross_poly_v2(coords, coords_num) <= 0.0f) ? 1 : -1;
   }
   else {
     /* check we're passing in correct args */
 #ifdef USE_STRICT_ASSERT
 #  ifndef NDEBUG
     if (coords_sign == 1) {
-      BLI_assert(cross_poly_v2(coords, coords_num) >= 0.0f);
+      BLI_assert(cross_poly_v2(coords, coords_num) <= 0.0f);
     }
     else {
-      BLI_assert(cross_poly_v2(coords, coords_num) <= 0.0f);
+      BLI_assert(cross_poly_v2(coords, coords_num) >= 0.0f);
     }
 #  endif
 #endif

source/blender/blenlib/tests/BLI_math_geom_test.cc

@@ -114,3 +114,38 @@ TEST(math_geom, IsectPointQuad2D)
   EXPECT_EQ(-1, isect_point_quad_v2(corner2, quad_cw[0], quad_cw[1], quad_cw[2], quad_cw[3]));
   EXPECT_EQ(+1, isect_point_quad_v2(corner2, quad_ccw[0], quad_ccw[1], quad_ccw[2], quad_ccw[3]));
 }
+
+TEST(math_geom, CrossPoly)
+{
+  const float tri_cw_2d[3][2] = {{-1, 0}, {0, 1}, {1, 0}};
+  const float tri_cw_3d[3][3] = {{-1, 0}, {0, 1}, {1, 0}};
+
+  const float tri_ccw_2d[3][2] = {{1, 0}, {0, 1}, {-1, 0}};
+  const float tri_ccw_3d[3][3] = {{1, 0}, {0, 1}, {-1, 0}};
+
+  auto cross_tri_v3_as_float3 = [](const float(*poly)[3]) -> float3 {
+    float n[3];
+    cross_tri_v3(n, UNPACK3(poly));
+    return float3(n[0], n[1], n[2]);
+  };
+
+  auto cross_poly_v3_as_float3 = [](const float(*poly)[3]) -> float3 {
+    float n[3];
+    cross_poly_v3(n, poly, 3);
+    return float3(n[0], n[1], n[2]);
+  };
+
+  /* Clockwise. */
+  EXPECT_EQ(cross_tri_v3_as_float3(tri_cw_3d)[2], -2);
+  EXPECT_EQ(cross_tri_v2(UNPACK3(tri_cw_2d)), -2);
+
+  EXPECT_EQ(cross_poly_v3_as_float3(tri_cw_3d)[2], -2);
+  EXPECT_EQ(cross_poly_v2(tri_cw_2d, 3), -2);
+
+  /* Counter clockwise. */
+  EXPECT_EQ(cross_tri_v3_as_float3(tri_ccw_3d)[2], 2);
+  EXPECT_EQ(cross_tri_v2(UNPACK3(tri_ccw_2d)), 2);
+
+  EXPECT_EQ(cross_poly_v3_as_float3(tri_ccw_3d)[2], 2);
+  EXPECT_EQ(cross_poly_v2(tri_ccw_2d, 3), 2);
+}