From 15623b26b93d96508fd28afedba5efa5dcce60d0 Mon Sep 17 00:00:00 2001
From: Howard Trickey <howard.trickey@gmail.com>
Date: Sun, 27 Apr 2025 19:36:28 -0400
Subject: [PATCH] Fix #137968: Bad custom normals after Manifold boolean.

Custom normals are interpolated with the new Manifold boolean
code. In the other two solvers, there is no interplation for the
CD_PROP_INT16_2D type so custom normals were left at (0,0) when
an exact copy from the input corner isn't possible.
The new code for manifold first does a join mesh, which converts
custom normals to float3, which can be (and are) interpolated.
However, if the input face (or face fragment) has flipped normal
in the boolean output, then the custom normal should be flipped to.
The fix is to do that for attributes called "custom_normal".
The general case of a "normal-like" thing with a different name
remains unsolved, but I doubt that case comes up often.
---
 .../geometry/intern/mesh_boolean_manifold.cc  | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/source/blender/geometry/intern/mesh_boolean_manifold.cc b/source/blender/geometry/intern/mesh_boolean_manifold.cc
index adca4996554..ea4f2b3c07f 100644
--- a/source/blender/geometry/intern/mesh_boolean_manifold.cc
+++ b/source/blender/geometry/intern/mesh_boolean_manifold.cc
@@ -1266,6 +1266,8 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
   Vector<bke::GAttributeReader> readers;
   Vector<GVArraySpan> srcs;
   Vector<GMutableSpan> dsts;
+  /* For each index of srcs and dest, we need to know if it is a "normal"-like attribute. */
+  Vector<bool> is_normal_attribute;
   input_attrs.foreach_attribute([&](const bke::AttributeIter &iter) {
     if (iter.domain != bke::AttrDomain::Corner || ELEM(iter.name, ".corner_vert", ".corner_edge"))
     {
@@ -1281,6 +1283,7 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
     readers.append(input_attrs.lookup_or_default(iter.name, iter.domain, iter.data_type));
     srcs.append(*readers.last());
     dsts.append(writers.last().span);
+    is_normal_attribute.append(iter.name == "custom_normal");
   });
   /* Loop per source face, as there is an expensive weight calculation that needs to be done per
    * face. */
@@ -1305,6 +1308,7 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
                 return out_to_in_corner_map[c] == -1;
               }))
           {
+            /* We copied the attributes using the corner map before calling this function. */
             continue;
           }
           /* At least one output corner did not map to an input corner. */
@@ -1315,19 +1319,26 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
           const IndexRange in_face = input_faces[in_face_index];
           const Span<int> in_face_verts = input_corner_verts.slice(in_face);
           const int in_face_size = in_face.size();
+          const Span<int> out_face_verts = output_corner_verts.slice(out_face);
           weights.resize(in_face_size);
           cos_2d.resize(in_face_size);
           float(*cos_2d_p)[2] = reinterpret_cast<float(*)[2]>(cos_2d.data());
           const float3 axis_dominant = bke::mesh::face_normal_calc(input_vert_positions,
                                                                    in_face_verts);
           axis_dominant_v3_to_m3(axis_mat.ptr(), axis_dominant);
+          /* We also need to know if the output face has a flipped normal compared
+           * to the corresponding input face (used if we have custom normals).
+           */
+          const float3 out_face_normal = bke::mesh::face_normal_calc(output_vert_positions,
+                                                                     out_face_verts);
+          const bool face_is_flipped = math::dot(axis_dominant, out_face_normal) < 0.0;
           for (const int i : in_face_verts.index_range()) {
             const float3 &co = input_vert_positions[in_face_verts[i]];
             cos_2d[i] = (axis_mat * co).xy();
           }
           /* Now the loop to actually interpolate attributes of the new-vertex corners of the
            * output face. */
-          for (const int out_c : output_faces[out_face_index]) {
+          for (const int out_c : out_face) {
             const int in_c = out_to_in_corner_map[out_c];
             if (in_c != -1) {
               continue;
@@ -1340,6 +1351,7 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
             for (const int attr_index : dsts.index_range()) {
               const GSpan src = srcs[attr_index];
               GMutableSpan dst = dsts[attr_index];
+              const bool need_flip = face_is_flipped && is_normal_attribute[attr_index];
               const CPPType &type = dst.type();
               bke::attribute_math::convert_to_static_type(type, [&](auto dummy) {
                 using T = decltype(dummy);
@@ -1350,6 +1362,12 @@ static void interpolate_corner_attributes(bke::MutableAttributeAccessor &output_
                   mixer.mix_in(0, src_typed[in_face[i]], weights[i]);
                 }
                 mixer.finalize();
+                if (need_flip) {
+                  /* The joined mesh has converted custom normals to float3. */
+                  if (type.is<float3>()) {
+                    dst.typed<float3>()[out_c] = -dst.typed<float3>()[out_c];
+                  }
+                }
               });
             }
           }