Fix #124836: Triangulate modifier changed behavior with custom normals

This reverts commit f3c32a36bc and the two followups.

The commit caused issues with both the operator and the modifier.

The operator could be fixed, for the modifier this needs deeper
investigation (see #124836 for a bit more info on this).

Until a better solution is found it is just better to go back to
previous behavior.

Reintroduces  #103562 for now

Pull Request: https://projects.blender.org/blender/blender/pulls/125499

source/blender/bmesh/tools/bmesh_triangulate.cc

@@ -90,8 +90,6 @@ void BM_mesh_triangulate(BMesh *bm,
   MemArena *pf_arena;
   Heap *pf_heap;
 
-  BM_custom_loop_normals_to_vector_layer(bm);
-
   pf_arena = BLI_memarena_new(BLI_POLYFILL_ARENA_SIZE, __func__);
 
   if (ngon_method == MOD_TRIANGULATE_NGON_BEAUTY) {
@@ -152,8 +150,6 @@ void BM_mesh_triangulate(BMesh *bm,
 
   BLI_memarena_free(pf_arena);
 
-  BM_custom_loop_normals_from_vector_layer(bm, false);
-
   if (ngon_method == MOD_TRIANGULATE_NGON_BEAUTY) {
     BLI_heap_free(pf_heap, nullptr);
   }

source/blender/editors/mesh/editmesh_tools.cc

@@ -5157,6 +5157,8 @@ static int edbm_quads_convert_to_tris_exec(bContext *C, wmOperator *op)
     BMOIter oiter;
     BMFace *f;
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     EDBM_op_init(em,
                  &bmop,
                  op,
@@ -5181,6 +5183,8 @@ static int edbm_quads_convert_to_tris_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBMUpdate_Params params{};
     params.calc_looptris = true;
     params.calc_normals = false;

source/blender/makesdna/DNA_modifier_types.h

@@ -1898,11 +1898,12 @@ typedef struct TriangulateModifierData {
 } TriangulateModifierData;
 
 /** #TriangulateModifierData.flag */
-#ifdef DNA_DEPRECATED_ALLOW
 enum {
+#ifdef DNA_DEPRECATED_ALLOW
   MOD_TRIANGULATE_BEAUTY = (1 << 0), /* deprecated */
-};
 #endif
+  MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS = 1 << 1,
+};
 
 /** #TriangulateModifierData.ngon_method triangulate method (N-gons). */
 enum {

source/blender/makesrna/intern/rna_modifier.cc

@@ -6623,6 +6623,16 @@ static void rna_def_modifier_triangulate(BlenderRNA *brna)
       "Triangulate only polygons with vertex count greater than or equal to this number");
   RNA_def_property_update(prop, 0, "rna_Modifier_update");
 
+  prop = RNA_def_property(srna, "keep_custom_normals", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, nullptr, "flag", MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS);
+  RNA_def_property_ui_text(
+      prop,
+      "Keep Normals",
+      "Try to preserve custom normals.\n"
+      "Warning: Depending on chosen triangulation method, "
+      "shading may not be fully preserved, \"Fixed\" method usually gives the best result here");
+  RNA_def_property_update(prop, 0, "rna_Modifier_update");
+
   RNA_define_lib_overridable(false);
 }
 

source/blender/modifiers/intern/MOD_triangulate.cc

@@ -33,27 +33,46 @@
 static Mesh *triangulate_mesh(Mesh *mesh,
                               const int quad_method,
                               const int ngon_method,
-                              const int min_vertices)
+                              const int min_vertices,
+                              const int flag)
 {
+  Mesh *result;
+  BMesh *bm;
   CustomData_MeshMasks cd_mask_extra{};
   cd_mask_extra.vmask = CD_MASK_ORIGINDEX;
   cd_mask_extra.emask = CD_MASK_ORIGINDEX;
   cd_mask_extra.pmask = CD_MASK_ORIGINDEX;
 
+  bool keep_clnors = (flag & MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS) != 0;
+
+  if (keep_clnors) {
+    void *data = CustomData_add_layer(
+        &mesh->corner_data, CD_NORMAL, CD_CONSTRUCT, mesh->corners_num);
+    memcpy(data, mesh->corner_normals().data(), mesh->corner_normals().size_in_bytes());
+    cd_mask_extra.lmask |= CD_MASK_NORMAL;
+  }
+
   BMeshCreateParams bmesh_create_params{};
   BMeshFromMeshParams bmesh_from_mesh_params{};
   bmesh_from_mesh_params.calc_face_normal = true;
   bmesh_from_mesh_params.calc_vert_normal = false;
   bmesh_from_mesh_params.cd_mask_extra = cd_mask_extra;
 
-  BMesh *bm = BKE_mesh_to_bmesh_ex(mesh, &bmesh_create_params, &bmesh_from_mesh_params);
+  bm = BKE_mesh_to_bmesh_ex(mesh, &bmesh_create_params, &bmesh_from_mesh_params);
 
   BM_mesh_triangulate(
       bm, quad_method, ngon_method, min_vertices, false, nullptr, nullptr, nullptr);
 
-  Mesh *result = BKE_mesh_from_bmesh_for_eval_nomain(bm, &cd_mask_extra, mesh);
+  result = BKE_mesh_from_bmesh_for_eval_nomain(bm, &cd_mask_extra, mesh);
   BM_mesh_free(bm);
 
+  if (keep_clnors) {
+    float(*corner_normals)[3] = static_cast<float(*)[3]>(
+        CustomData_get_layer_for_write(&result->corner_data, CD_NORMAL, result->corners_num));
+    BKE_mesh_set_custom_normals(result, corner_normals);
+    CustomData_free_layers(&result->corner_data, CD_NORMAL, result->corners_num);
+  }
+
   return result;
 }
 
@@ -73,7 +92,9 @@ static Mesh *modify_mesh(ModifierData *md, const ModifierEvalContext * /*ctx*/,
 {
   TriangulateModifierData *tmd = (TriangulateModifierData *)md;
   Mesh *result;
-  if (!(result = triangulate_mesh(mesh, tmd->quad_method, tmd->ngon_method, tmd->min_vertices))) {
+  if (!(result = triangulate_mesh(
+            mesh, tmd->quad_method, tmd->ngon_method, tmd->min_vertices, tmd->flag)))
+  {
     return mesh;
   }
 
@@ -92,6 +113,7 @@ static void panel_draw(const bContext * /*C*/, Panel *panel)
   uiItemR(layout, ptr, "quad_method", UI_ITEM_NONE, nullptr, ICON_NONE);
   uiItemR(layout, ptr, "ngon_method", UI_ITEM_NONE, nullptr, ICON_NONE);
   uiItemR(layout, ptr, "min_vertices", UI_ITEM_NONE, nullptr, ICON_NONE);
+  uiItemR(layout, ptr, "keep_custom_normals", UI_ITEM_NONE, nullptr, ICON_NONE);
 
   modifier_panel_end(layout, ptr);
 }