Fix #108105: Fix operators affecting hidden geometry

These operators in sculpt mode affect hidden geometry which is undesirable.
- bpy.ops.sculpt.face_set_edit
- bpy.ops.sculpt.face_sets_init
- bpy.ops.sculpt.face_sets_create(mode='SELECTION')
- bpy.ops.mesh.paint_mask_extract()
- bpy.ops.mesh.paint_mask_slice()

This PR adds checks for hidden faces in these operators.

For operator `bpy.ops.sculpt.face_sets_init` it also modifies the way
face sets indices were generated so generated indices is unique. This
is needed so new initialized face sets do not get same indices as hidden
face sets.

For generating unique face set index I have created a set container
which stores hidden face sets indices. Before assigning indices to a
face set it checks if it is already in set container, if it is then
it'll keep increasing index by 1 until it is not in set container.

Modifying Operator `bpy.ops.mesh.paint_mask_slice()` is little complex,
it will not affect hidden geometry only when fill holes option is
unchecked, because currently filling hole code does not take hidden
geometry into account and because of this in some cases hidden part
of geometry can be trapped inside mesh.

Co-authored-by: Hans Goudey <hans@blender.org>
Pull Request: https://projects.blender.org/blender/blender/pulls/119168

source/blender/editors/mesh/editmesh_mask_extract.cc

@@ -246,6 +246,10 @@ static void geometry_extract_tag_masked_faces(BMesh *bm, GeometryExtractParams *
     bool keep_face = true;
     BMVert *v;
     BMIter face_iter;
+    if (BM_elem_flag_test_bool(f, BM_ELEM_HIDDEN)) {
+      BM_elem_flag_set(f, BM_ELEM_TAG, true);
+      continue;
+    };
     BM_ITER_ELEM (v, &face_iter, f, BM_VERTS_OF_FACE) {
       const float mask = BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
       if (mask < threshold) {
@@ -417,6 +421,10 @@ static void slice_paint_mask(BMesh *bm, bool invert, bool fill_holes, float mask
         break;
       }
     }
+    if (BM_elem_flag_test_bool(f, BM_ELEM_HIDDEN)) {
+      keep_face = false;
+    };
+    /* This invert behavior is fragile, as it potentially marks faces which are hidden */
     if (invert) {
       keep_face = !keep_face;
     }

source/blender/editors/sculpt_paint/sculpt_face_set.cc

@@ -672,9 +672,15 @@ static int create_op_exec(bContext *C, wmOperator *op)
     case CreateMode::Selection: {
       const VArraySpan<bool> select_poly = *attributes.lookup_or_default<bool>(
           ".select_poly", bke::AttrDomain::Face, false);
+      const VArraySpan<bool> hide_poly = *attributes.lookup<bool>(".hide_poly",
+                                                                  bke::AttrDomain::Face);
+
       face_sets_update(object, nodes, [&](const Span<int> indices, MutableSpan<int> face_sets) {
         for (const int i : indices.index_range()) {
           if (select_poly[indices[i]]) {
+            if (!hide_poly.is_empty() && hide_poly[i]) {
+              continue;
+            }
             face_sets[i] = next_face_set;
           }
         }
@@ -759,14 +765,24 @@ static void init_flood_fill(Object &ob, const FaceSetsFloodFillFn &test_fn)
         faces, corner_edges, edges.size(), ss.edge_to_face_offsets, ss.edge_to_face_indices);
   }
 
+  const bke::AttributeAccessor attributes = mesh->attributes();
+  const VArraySpan<bool> hide_poly = *attributes.lookup<bool>(".hide_poly", bke::AttrDomain::Face);
+  const Set<int> hidden_face_sets = gather_hidden_face_sets(hide_poly, face_sets.span);
+
   int next_face_set = 1;
 
   for (const int i : faces.index_range()) {
+    if (!hide_poly.is_empty() && hide_poly[i]) {
+      continue;
+    }
     if (visited_faces[i]) {
       continue;
     }
     std::queue<int> queue;
 
+    while (hidden_face_sets.contains(next_face_set)) {
+      next_face_set += 1;
+    }
     face_sets.span[i] = next_face_set;
     visited_faces[i].set(true);
     queue.push(i);
@@ -783,6 +799,9 @@ static void init_flood_fill(Object &ob, const FaceSetsFloodFillFn &test_fn)
           if (visited_faces[neighbor_i]) {
             continue;
           }
+          if (!hide_poly.is_empty() && hide_poly[neighbor_i]) {
+            continue;
+          }
           if (!test_fn(face_i, edge_i, neighbor_i)) {
             continue;
           }
@@ -800,6 +819,22 @@ static void init_flood_fill(Object &ob, const FaceSetsFloodFillFn &test_fn)
   face_sets.finish();
 }
 
+Set<int> gather_hidden_face_sets(const Span<bool> hide_poly, const Span<int> face_sets)
+{
+  if (hide_poly.is_empty()) {
+    return {};
+  }
+
+  Set<int> hidden_face_sets;
+  for (const int i : hide_poly.index_range()) {
+    if (hide_poly[i]) {
+      hidden_face_sets.add(face_sets[i]);
+    }
+  }
+
+  return hidden_face_sets;
+}
+
 static int init_op_exec(bContext *C, wmOperator *op)
 {
   Object &ob = *CTX_data_active_object(C);
@@ -851,8 +886,25 @@ static int init_op_exec(bContext *C, wmOperator *op)
       bke::SpanAttributeWriter<int> face_sets = ensure_face_sets_mesh(ob);
       const VArraySpan<int> material_indices = *attributes.lookup_or_default<int>(
           "material_index", bke::AttrDomain::Face, 0);
+      const VArraySpan<bool> hide_poly = *attributes.lookup<bool>(".hide_poly",
+                                                                  bke::AttrDomain::Face);
+      const Set<int> hidden_face_sets = gather_hidden_face_sets(hide_poly, face_sets.span);
+
+      int prev_material = material_indices[0];
+      int material_face_set = 1;
       for (const int i : IndexRange(mesh->faces_num)) {
-        face_sets.span[i] = material_indices[i] + 1;
+        if (!hide_poly.is_empty() && hide_poly[i]) {
+          continue;
+        }
+        if (prev_material != material_indices[i]) {
+          material_face_set += 1;
+        }
+        while (hidden_face_sets.contains(material_face_set)) {
+          material_face_set += 1;
+        }
+
+        face_sets.span[i] = material_face_set;
+        prev_material = material_indices[i];
       }
       face_sets.finish();
       break;
@@ -1656,7 +1708,7 @@ void SCULPT_OT_face_sets_edit(wmOperatorType *ot)
 
   ot->prop = RNA_def_boolean(ot->srna,
                              "modify_hidden",
-                             true,
+                             false,
                              "Modify Hidden",
                              "Apply the edit operation to hidden geometry");
 }

source/blender/editors/sculpt_paint/sculpt_intern.hh

@@ -974,6 +974,7 @@ bool vert_has_unique_face_set(const SculptSession &ss, PBVHVertRef vertex);
 bke::SpanAttributeWriter<int> ensure_face_sets_mesh(Object &object);
 int ensure_face_sets_bmesh(Object &object);
 Array<int> duplicate_face_sets(const Mesh &mesh);
+Set<int> gather_hidden_face_sets(Span<bool> hide_poly, Span<int> face_sets);
 
 }