Sculpt: Add increase / decrease visibility operator

This PR adds a new operator: `PAINT_OT_visibility_edit` to support
iteratively expanding or shrinking the visibility of a mesh, similar to
the *Grow / Shrink Mask* actions and the *Grow / Shrink Face Set*
options. This operator is exposed via two new entries in the *Sculpt*
toolbar entry as *Show More* and *Show Less* and have also been
assigned to Page Up and Page Down in the default Blender keybinds for
Sculpt Mode.

### Technical Details
Each of the PBVH types is solved slightly differently, though the
general principle for each is as follows:

1. Make a copy of the current mesh visibility state
2. Iterate over elements (faces & corners if available, otherwise
   vertices) to look at adjacency information
3. Apply appropriate visibility change to vertices
4. Sync face visibility

### Limitations
* Currently, like all other operators in the `paint_hide.cc` file. This
  new operator is limited to Sculpt mode only.

Based off of [this](https://blender.community/c/rightclickselect/pz4y/)
RCS request.

Pull Request: https://projects.blender.org/blender/blender/pulls/120282

scripts/presets/keyconfig/keymap_data/blender_default.py

@@ -5619,6 +5619,10 @@ def km_sculpt(params):
          {"properties": [("mode", 'HIDE_ACTIVE')]}),
         ("paint.hide_show_all", {"type": 'H', "value": 'PRESS', "alt": True},
          {"properties": [("action", "SHOW")]}),
+        ("paint.visibility_filter", {"type": 'PAGE_UP', "value": 'PRESS', "repeat": True},
+         {"properties": [("action", 'GROW')]}),
+        ("paint.visibility_filter", {"type": 'PAGE_DOWN', "value": 'PRESS', "repeat": True},
+         {"properties": [("action", 'SHRINK')]}),
         ("sculpt.face_set_edit", {"type": 'W', "value": 'PRESS', "ctrl": True},
          {"properties": [("mode", 'GROW')]}),
         ("sculpt.face_set_edit", {"type": 'W', "value": 'PRESS', "ctrl": True, "alt": True},

scripts/startup/bl_ui/space_view3d.py

@@ -3819,6 +3819,12 @@ class VIEW3D_MT_sculpt(Menu):
         props = layout.operator("paint.hide_show_masked", text="Hide Masked")
         props.action = 'HIDE'
 
+        props = layout.operator("paint.visibility_filter", text="Grow Visibility")
+        props.action = "GROW"
+
+        props = layout.operator("paint.visibility_filter", text="Shrink Visibility")
+        props.action = "SHRINK"
+
         layout.separator()
 
         props = layout.operator("sculpt.trim_box_gesture", text="Box Trim")

source/blender/blenkernel/BKE_mesh.hh

@@ -343,6 +343,12 @@ void mesh_sharp_edges_set_from_angle(Mesh &mesh, float angle, bool keep_sharp_ed
  * vertices are hidden. */
 void mesh_edge_hide_from_vert(Span<int2> edges, Span<bool> hide_vert, MutableSpan<bool> hide_edge);
 
+/* Hide faces when any of their vertices are hidden. */
+void mesh_face_hide_from_vert(OffsetIndices<int> faces,
+                              Span<int> corner_verts,
+                              Span<bool> hide_vert,
+                              MutableSpan<bool> hide_poly);
+
 /** Make edge and face visibility consistent with vertices. */
 void mesh_hide_vert_flush(Mesh &mesh);
 /** Make vertex and edge visibility consistent with faces. */

source/blender/blenkernel/intern/mesh_evaluate.cc

@@ -517,11 +517,10 @@ void mesh_edge_hide_from_vert(const Span<int2> edges,
   });
 }
 
-/* Hide faces when any of their vertices are hidden. */
-static void face_hide_from_vert(const OffsetIndices<int> faces,
-                                const Span<int> corner_verts,
-                                const Span<bool> hide_vert,
-                                MutableSpan<bool> hide_poly)
+void mesh_face_hide_from_vert(const OffsetIndices<int> faces,
+                              const Span<int> corner_verts,
+                              const Span<bool> hide_vert,
+                              MutableSpan<bool> hide_poly)
 {
   using namespace blender;
   threading::parallel_for(faces.index_range(), 4096, [&](const IndexRange range) {
@@ -552,7 +551,7 @@ void mesh_hide_vert_flush(Mesh &mesh)
       ".hide_poly", AttrDomain::Face);
 
   mesh_edge_hide_from_vert(mesh.edges(), hide_vert_span, hide_edge.span);
-  face_hide_from_vert(mesh.faces(), mesh.corner_verts(), hide_vert_span, hide_poly.span);
+  mesh_face_hide_from_vert(mesh.faces(), mesh.corner_verts(), hide_vert_span, hide_poly.span);
 
   hide_edge.finish();
   hide_poly.finish();

source/blender/editors/sculpt_paint/paint_hide.cc

@@ -192,7 +192,7 @@ enum class VisAction {
   Show = 1,
 };
 
-static bool action_to_hide(const VisAction action)
+constexpr static bool action_to_hide(const VisAction action)
 {
   return action == VisAction::Hide;
 }
@@ -253,6 +253,18 @@ static void flush_face_changes_node(Mesh &mesh,
   hide_poly.finish();
 }
 
+/* Updates a node's face's visibility based on the updated vertex visibility. */
+static void flush_face_changes(Mesh &mesh, const Span<bool> hide_vert)
+{
+  bke::MutableAttributeAccessor attributes = mesh.attributes_for_write();
+
+  bke::SpanAttributeWriter<bool> hide_poly = attributes.lookup_or_add_for_write_span<bool>(
+      ".hide_poly", bke::AttrDomain::Face);
+
+  bke::mesh_face_hide_from_vert(mesh.faces(), mesh.corner_verts(), hide_vert, hide_poly.span);
+  hide_poly.finish();
+}
+
 /* Updates all of a mesh's edge visibility based on vertex visibility. */
 static void flush_edge_changes(Mesh &mesh, const Span<bool> hide_vert)
 {
@@ -787,6 +799,402 @@ void PAINT_OT_visibility_invert(wmOperatorType *ot)
   ot->flag = OPTYPE_REGISTER;
 }
 
+/* Number of vertices per iteration step size when growing or shrinking visibility. */
+static constexpr float VERTEX_ITERATION_THRESHOLD = 50000.0f;
+
+/* Extracting the loop and comparing against / writing with a constant `false` or `true` instead of
+ * using #action_to_hide results in a nearly 600ms speedup on a mesh with 1.5m verts. */
+template<bool value>
+static void affect_visibility_mesh(const IndexRange face,
+                                   const Span<int> corner_verts,
+                                   const Span<bool> read_buffer,
+                                   MutableSpan<bool> write_buffer)
+{
+  for (const int corner : face) {
+    int vert = corner_verts[corner];
+    if (read_buffer[vert] != value) {
+      continue;
+    }
+
+    const int prev = bke::mesh::face_corner_prev(face, corner);
+    const int prev_vert = corner_verts[prev];
+    write_buffer[prev_vert] = value;
+
+    const int next = bke::mesh::face_corner_next(face, corner);
+    const int next_vert = corner_verts[next];
+    write_buffer[next_vert] = value;
+  }
+}
+
+struct DualBuffer {
+  Array<bool> front;
+  Array<bool> back;
+
+  MutableSpan<bool> write_buffer(int count)
+  {
+    return count % 2 == 0 ? back.as_mutable_span() : front.as_mutable_span();
+  }
+
+  Span<bool> read_buffer(int count)
+  {
+    return count % 2 == 0 ? front.as_span() : back.as_span();
+  }
+};
+
+static void propagate_vertex_visibility(Mesh &mesh,
+                                        DualBuffer &buffers,
+                                        const VArraySpan<bool> &hide_poly,
+                                        const VisAction action,
+                                        const int iterations)
+{
+  const OffsetIndices faces = mesh.faces();
+  const Span<int> corner_verts = mesh.corner_verts();
+
+  for (const int i : IndexRange(iterations)) {
+    Span<bool> read_buffer = buffers.read_buffer(i);
+    MutableSpan<bool> write_buffer = buffers.write_buffer(i);
+    threading::parallel_for(faces.index_range(), 1024, [&](const IndexRange range) {
+      for (const int face_index : range) {
+        if (!hide_poly[face_index]) {
+          continue;
+        }
+        const IndexRange face = faces[face_index];
+        if (action == VisAction::Hide) {
+          affect_visibility_mesh<true>(face, corner_verts, read_buffer, write_buffer);
+        }
+        else {
+          affect_visibility_mesh<false>(face, corner_verts, read_buffer, write_buffer);
+        }
+      }
+    });
+
+    flush_face_changes(mesh, write_buffer);
+  }
+}
+
+static void update_undo_state(Object &object,
+                              const Span<PBVHNode *> nodes,
+                              const Span<bool> old_hide_vert,
+                              const Span<bool> new_hide_vert)
+{
+  threading::parallel_for(nodes.index_range(), 1, [&](const IndexRange range) {
+    for (PBVHNode *node : nodes.slice(range)) {
+      for (const int vert : bke::pbvh::node_unique_verts(*node)) {
+        if (old_hide_vert[vert] != new_hide_vert[vert]) {
+          undo::push_node(object, node, undo::Type::HideVert);
+          break;
+        }
+      }
+    }
+  });
+}
+
+static void update_node_visibility_from_face_changes(const Span<PBVHNode *> nodes,
+                                                     const Span<int> tri_faces,
+                                                     const Span<bool> orig_hide_poly,
+                                                     const Span<bool> new_hide_poly,
+                                                     const Span<bool> hide_vert)
+{
+  threading::EnumerableThreadSpecific<Vector<int>> all_face_indices;
+  threading::parallel_for(nodes.index_range(), 1, [&](const IndexRange range) {
+    Vector<int> &face_indices = all_face_indices.local();
+    for (PBVHNode *node : nodes.slice(range)) {
+      bool any_changed = false;
+      const Span<int> indices = bke::pbvh::node_face_indices_calc_mesh(
+          tri_faces, *node, face_indices);
+      for (const int face_index : indices) {
+        if (orig_hide_poly[face_index] != new_hide_poly[face_index]) {
+          any_changed = true;
+          break;
+        }
+      }
+
+      if (any_changed) {
+        BKE_pbvh_node_mark_update_visibility(node);
+        bke::pbvh::node_update_visibility_mesh(hide_vert, *node);
+      }
+    }
+  });
+}
+
+static void grow_shrink_visibility_mesh(Object &object,
+                                        const Span<PBVHNode *> nodes,
+                                        const VisAction action,
+                                        const int iterations)
+{
+  Mesh &mesh = *static_cast<Mesh *>(object.data);
+  bke::MutableAttributeAccessor attributes = mesh.attributes_for_write();
+  if (!attributes.contains(".hide_vert")) {
+    /* If the entire mesh is visible, we can neither grow nor shrink the boundary. */
+    return;
+  }
+
+  bke::SpanAttributeWriter<bool> hide_vert = attributes.lookup_or_add_for_write_span<bool>(
+      ".hide_vert", bke::AttrDomain::Point);
+  const VArraySpan hide_poly = *attributes.lookup_or_default<bool>(
+      ".hide_poly", bke::AttrDomain::Face, false);
+
+  DualBuffer buffers;
+  buffers.back.reinitialize(hide_vert.span.size());
+  buffers.front.reinitialize(hide_vert.span.size());
+  array_utils::copy(hide_vert.span.as_span(), buffers.back.as_mutable_span());
+  array_utils::copy(hide_vert.span.as_span(), buffers.front.as_mutable_span());
+
+  Array<bool> orig_hide_poly(hide_poly);
+  propagate_vertex_visibility(mesh, buffers, hide_poly, action, iterations);
+
+  const Span<bool> last_buffer = buffers.write_buffer(iterations - 1);
+
+  update_undo_state(object, nodes, hide_vert.span, last_buffer);
+
+  /* We can wait until after all iterations are done to flush edge changes as they are
+   * not used for coarse filtering while iterating.*/
+  flush_edge_changes(mesh, last_buffer);
+
+  update_node_visibility_from_face_changes(
+      nodes, mesh.corner_tri_faces(), orig_hide_poly, hide_poly, last_buffer);
+  array_utils::copy(last_buffer, hide_vert.span);
+  hide_vert.finish();
+}
+
+/* TODO: This is probably better off as a member function of a CCGKey?*/
+static int elem_xy_to_index(int x, int y, int grid_size)
+{
+  return y * grid_size + x;
+}
+
+struct DualBitBuffer {
+  BitGroupVector<> front;
+  BitGroupVector<> back;
+
+  BitGroupVector<> &write_buffer(int count)
+  {
+    return count % 2 == 0 ? back : front;
+  }
+
+  BitGroupVector<> &read_buffer(int count)
+  {
+    return count % 2 == 0 ? front : back;
+  }
+};
+
+static void grow_shrink_visibility_grid(Depsgraph &depsgraph,
+                                        Object &object,
+                                        PBVH &pbvh,
+                                        const Span<PBVHNode *> nodes,
+                                        const VisAction action,
+                                        const int iterations)
+{
+  Mesh &mesh = *static_cast<Mesh *>(object.data);
+  SubdivCCG &subdiv_ccg = *object.sculpt->subdiv_ccg;
+
+  BitGroupVector<> &grid_hidden = BKE_subdiv_ccg_grid_hidden_ensure(subdiv_ccg);
+
+  const bool desired_state = action_to_hide(action);
+  const CCGKey key = *BKE_pbvh_get_grid_key(pbvh);
+
+  DualBitBuffer buffers;
+  buffers.front = grid_hidden;
+  buffers.back = grid_hidden;
+
+  Array<bool> node_changed(nodes.size(), false);
+
+  for (const int i : IndexRange(iterations)) {
+    BitGroupVector<> &read_buffer = buffers.read_buffer(i);
+    BitGroupVector<> &write_buffer = buffers.write_buffer(i);
+
+    threading::parallel_for(nodes.index_range(), 1, [&](const IndexRange range) {
+      for (const int node_index : range) {
+        PBVHNode *node = nodes[node_index];
+        const Span<int> grids = bke::pbvh::node_grid_indices(*node);
+
+        for (const int grid_index : grids) {
+          for (const int y : IndexRange(key.grid_size)) {
+            for (const int x : IndexRange(key.grid_size)) {
+              const int grid_elem_idx = elem_xy_to_index(x, y, key.grid_size);
+              if (read_buffer[grid_index][grid_elem_idx] != desired_state) {
+                continue;
+              }
+
+              SubdivCCGCoord coord{};
+              coord.grid_index = grid_index;
+              coord.x = x;
+              coord.y = y;
+
+              SubdivCCGNeighbors neighbors;
+              BKE_subdiv_ccg_neighbor_coords_get(subdiv_ccg, coord, true, neighbors);
+
+              for (const int j : neighbors.coords.index_range()) {
+                const SubdivCCGCoord neighbor = neighbors.coords[j];
+                const int neighbor_grid_elem_idx = elem_xy_to_index(
+                    neighbor.x, neighbor.y, key.grid_size);
+
+                write_buffer[neighbor.grid_index][neighbor_grid_elem_idx].set(desired_state);
+              }
+            }
+          }
+        }
+
+        node_changed[node_index] = true;
+      }
+    });
+  }
+
+  IndexMaskMemory memory;
+  IndexMask mask = IndexMask::from_bools(node_changed, memory);
+  mask.foreach_index(GrainSize(1), [&](const int64_t index) {
+    undo::push_node(object, nodes[index], undo::Type::HideVert);
+  });
+
+  BitGroupVector<> &last_buffer = buffers.write_buffer(iterations - 1);
+  grid_hidden = std::move(last_buffer);
+
+  threading::parallel_for(nodes.index_range(), 1, [&](const IndexRange range) {
+    for (const int node_index : range) {
+      if (!node_changed[node_index]) {
+        continue;
+      }
+      PBVHNode *node = nodes[node_index];
+
+      BKE_pbvh_node_mark_update_visibility(node);
+      bke::pbvh::node_update_visibility_grids(grid_hidden, *node);
+    }
+  });
+
+  multires_mark_as_modified(&depsgraph, &object, MULTIRES_HIDDEN_MODIFIED);
+  BKE_pbvh_sync_visibility_from_verts(pbvh, &mesh);
+}
+
+static Array<bool> duplicate_visibility_bmesh(const Object &object)
+{
+  const SculptSession &ss = *object.sculpt;
+  BMesh &bm = *ss.bm;
+  Array<bool> result(bm.totvert);
+  BM_mesh_elem_table_ensure(&bm, BM_VERT);
+  for (const int i : result.index_range()) {
+    result[i] = BM_elem_flag_test_bool(BM_vert_at_index(&bm, i), BM_ELEM_HIDDEN);
+  }
+  return result;
+}
+
+static void grow_shrink_visibility_bmesh(Object &object,
+                                         PBVH &pbvh,
+                                         const Span<PBVHNode *> nodes,
+                                         const VisAction action,
+                                         const int iterations)
+{
+
+  SculptSession &ss = *object.sculpt;
+
+  for (const int i : IndexRange(iterations)) {
+    UNUSED_VARS(i);
+    const Array<bool> prev_visibility = duplicate_visibility_bmesh(object);
+    partialvis_update_bmesh_nodes(object, nodes, action, [&](const BMVert *vert) {
+      int vi = BM_elem_index_get(vert);
+      PBVHVertRef vref = BKE_pbvh_index_to_vertex(pbvh, vi);
+      SculptVertexNeighborIter ni;
+
+      bool should_change = false;
+      SCULPT_VERTEX_NEIGHBORS_ITER_BEGIN (ss, vref, ni) {
+        if (prev_visibility[ni.index] == action_to_hide(action)) {
+          /* Not returning instantly to avoid leaking memory. */
+          should_change = true;
+          break;
+        }
+      }
+      SCULPT_VERTEX_NEIGHBORS_ITER_END(ni);
+      return should_change;
+    });
+  }
+}
+
+static int visibility_filter_exec(bContext *C, wmOperator *op)
+{
+  Object &object = *CTX_data_active_object(C);
+  Depsgraph &depsgraph = *CTX_data_ensure_evaluated_depsgraph(C);
+
+  PBVH &pbvh = *BKE_sculpt_object_pbvh_ensure(&depsgraph, &object);
+  BLI_assert(BKE_object_sculpt_pbvh_get(&object) == &pbvh);
+
+  const VisAction mode = VisAction(RNA_enum_get(op->ptr, "action"));
+
+  Vector<PBVHNode *> nodes = bke::pbvh::search_gather(pbvh, {});
+
+  const SculptSession &ss = *object.sculpt;
+  int num_verts = SCULPT_vertex_count_get(ss);
+
+  int iterations = RNA_int_get(op->ptr, "iterations");
+
+  if (RNA_boolean_get(op->ptr, "auto_iteration_count")) {
+    /* Automatically adjust the number of iterations based on the number
+     * of vertices in the mesh. */
+    iterations = int(num_verts / VERTEX_ITERATION_THRESHOLD) + 1;
+  }
+
+  undo::push_begin(object, op);
+  switch (BKE_pbvh_type(pbvh)) {
+    case PBVH_FACES:
+      grow_shrink_visibility_mesh(object, nodes, mode, iterations);
+      break;
+    case PBVH_GRIDS:
+      grow_shrink_visibility_grid(depsgraph, object, pbvh, nodes, mode, iterations);
+      break;
+    case PBVH_BMESH:
+      grow_shrink_visibility_bmesh(object, pbvh, nodes, mode, iterations);
+      break;
+  }
+  undo::push_end(object);
+
+  SCULPT_topology_islands_invalidate(*object.sculpt);
+  tag_update_visibility(*C);
+
+  return OPERATOR_FINISHED;
+}
+
+void PAINT_OT_visibility_filter(wmOperatorType *ot)
+{
+  static EnumPropertyItem actions[] = {
+      {int(VisAction::Show),
+       "GROW",
+       0,
+       "Grow Visibility",
+       "Grow the visibility by one face based on mesh topology"},
+      {int(VisAction::Hide),
+       "SHRINK",
+       0,
+       "Shrink Visibility",
+       "Shrink the visibility by one face based on mesh topology"},
+      {0, nullptr, 0, nullptr, nullptr},
+  };
+
+  ot->name = "Visibility Filter";
+  ot->idname = "PAINT_OT_visibility_filter";
+  ot->description = "Edit the visibility of the current mesh";
+
+  ot->exec = visibility_filter_exec;
+  ot->poll = SCULPT_mode_poll_view3d;
+
+  ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+
+  RNA_def_enum(ot->srna, "action", actions, int(VisAction::Show), "Action", "");
+
+  RNA_def_int(ot->srna,
+              "iterations",
+              1,
+              1,
+              100,
+              "Iterations",
+              "Number of times that the filter is going to be applied",
+              1,
+              100);
+  RNA_def_boolean(
+      ot->srna,
+      "auto_iteration_count",
+      true,
+      "Auto Iteration Count",
+      "Use an automatic number of iterations based on the number of vertices of the sculpt");
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */

source/blender/editors/sculpt_paint/paint_intern.hh

@@ -461,6 +461,7 @@ void PAINT_OT_hide_show_line_gesture(wmOperatorType *ot);
 void PAINT_OT_hide_show_polyline_gesture(wmOperatorType *ot);
 
 void PAINT_OT_visibility_invert(wmOperatorType *ot);
+void PAINT_OT_visibility_filter(wmOperatorType *ot);
 }  // namespace blender::ed::sculpt_paint::hide
 
 /* `paint_mask.cc` */

source/blender/editors/sculpt_paint/paint_ops.cc

@@ -1561,6 +1561,7 @@ void ED_operatortypes_paint()
   WM_operatortype_append(hide::PAINT_OT_hide_show_line_gesture);
   WM_operatortype_append(hide::PAINT_OT_hide_show_polyline_gesture);
   WM_operatortype_append(hide::PAINT_OT_visibility_invert);
+  WM_operatortype_append(hide::PAINT_OT_visibility_filter);
 
   /* paint masking */
   WM_operatortype_append(mask::PAINT_OT_mask_flood_fill);