Refactor: Sculpt: Reduce reliance on BVH tree geometry back pointers

Retrieve the relevant data directly from the original object's mesh
(or the evaluated SubdivCCG in that case) rather than using the
BVH tree's geometry points which we'd like to remove.

Pull Request: https://projects.blender.org/blender/blender/pulls/126284

source/blender/blenkernel/BKE_pbvh_api.hh

@@ -533,10 +533,6 @@ void BKE_pbvh_node_get_bm_orco_data(blender::bke::pbvh::Node *node,
                                     float (**r_orco_coords)[3],
                                     BMVert ***r_orco_verts);
 
-bool pbvh_has_mask(const blender::bke::pbvh::Tree &pbvh);
-
-bool pbvh_has_face_sets(blender::bke::pbvh::Tree &pbvh);
-
 blender::Span<blender::float3> BKE_pbvh_get_vert_positions(const blender::bke::pbvh::Tree &pbvh);
 blender::MutableSpan<blender::float3> BKE_pbvh_get_vert_positions(blender::bke::pbvh::Tree &pbvh);
 blender::Span<blender::float3> BKE_pbvh_get_vert_normals(const blender::bke::pbvh::Tree &pbvh);

source/blender/blenkernel/intern/pbvh.cc

@@ -2690,33 +2690,6 @@ bool BKE_pbvh_is_deformed(const blender::bke::pbvh::Tree &pbvh)
   return pbvh.deformed_;
 }
 
-bool pbvh_has_mask(const blender::bke::pbvh::Tree &pbvh)
-{
-  switch (pbvh.type()) {
-    case blender::bke::pbvh::Type::Grids:
-      return BKE_subdiv_ccg_key_top_level(*pbvh.subdiv_ccg_).has_mask;
-    case blender::bke::pbvh::Type::Mesh:
-      return pbvh.mesh_->attributes().contains(".sculpt_mask");
-    case blender::bke::pbvh::Type::BMesh:
-      return pbvh.bm_ &&
-             CustomData_has_layer_named(&pbvh.bm_->vdata, CD_PROP_FLOAT, ".sculpt_mask");
-  }
-
-  return false;
-}
-
-bool pbvh_has_face_sets(blender::bke::pbvh::Tree &pbvh)
-{
-  switch (pbvh.type()) {
-    case blender::bke::pbvh::Type::Grids:
-    case blender::bke::pbvh::Type::Mesh:
-      return pbvh.mesh_->attributes().contains(".sculpt_face_set");
-    case blender::bke::pbvh::Type::BMesh:
-      return CustomData_has_layer_named(&pbvh.bm_->pdata, CD_PROP_FLOAT, ".sculpt_mask");
-  }
-  return false;
-}
-
 namespace blender::bke::pbvh {
 
 void set_frustum_planes(Tree &pbvh, PBVHFrustumPlanes *planes)

source/blender/draw/engines/overlay/overlay_sculpt.cc

@@ -11,10 +11,14 @@
 #include "draw_cache_impl.hh"
 #include "overlay_private.hh"
 
+#include "BKE_attribute.hh"
+#include "BKE_mesh.hh"
 #include "BKE_paint.hh"
 #include "BKE_pbvh_api.hh"
 #include "BKE_subdiv_ccg.hh"
 
+#include "DEG_depsgraph_query.hh"
+
 void OVERLAY_sculpt_cache_init(OVERLAY_Data *vedata)
 {
   OVERLAY_PassList *psl = vedata->psl;
@@ -37,7 +41,8 @@ void OVERLAY_sculpt_cache_populate(OVERLAY_Data *vedata, Object *ob)
   OVERLAY_PrivateData *pd = vedata->stl->pd;
   const DRWContextState *draw_ctx = DRW_context_state_get();
   blender::gpu::Batch *sculpt_overlays;
-  blender::bke::pbvh::Tree *pbvh = ob->sculpt->pbvh.get();
+  const SculptSession &ss = *ob->sculpt;
+  blender::bke::pbvh::Tree *pbvh = ss.pbvh.get();
 
   const bool use_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->rv3d);
 
@@ -47,13 +52,46 @@ void OVERLAY_sculpt_cache_populate(OVERLAY_Data *vedata, Object *ob)
     return;
   }
 
-  if (!pbvh_has_mask(*pbvh) && !pbvh_has_face_sets(*pbvh)) {
-    /* The SculptSession and the pbvh::Tree can be created without a Mask data-layer or Face Set
-     * data-layer. (masks data-layers are created after using a mask tool), so in these cases there
-     * is nothing to draw. */
+  /* Using the original object/geometry is necessary because we skip depsgraph updates in sculpt
+   * mode to improve performance. This means the evaluated mesh doesn't have the latest face set,
+   * visibility, and mask data. */
+  Object *object_orig = reinterpret_cast<Object *>(DEG_get_original_id(&ob->id));
+  if (!object_orig) {
+    BLI_assert_unreachable();
     return;
   }
 
+  switch (pbvh->type()) {
+    case blender::bke::pbvh::Type::Mesh: {
+      const Mesh &mesh = *static_cast<const Mesh *>(object_orig->data);
+      if (!mesh.attributes().contains(".sculpt_face_set") &&
+          !mesh.attributes().contains(".sculpt_mask"))
+      {
+        return;
+      }
+      break;
+    }
+    case blender::bke::pbvh::Type::Grids: {
+      const SubdivCCG &subdiv_ccg = *ss.subdiv_ccg;
+      const Mesh &base_mesh = *static_cast<const Mesh *>(object_orig->data);
+      if (BKE_subdiv_ccg_key_top_level(subdiv_ccg).has_mask &&
+          !base_mesh.attributes().contains(".sculpt_face_set"))
+      {
+        return;
+      }
+      break;
+    }
+    case blender::bke::pbvh::Type::BMesh: {
+      const BMesh &bm = *ss.bm;
+      if (!CustomData_has_layer_named(&bm.pdata, CD_PROP_FLOAT, ".sculpt_face_set") &&
+          !CustomData_has_layer_named(&bm.pdata, CD_PROP_FLOAT, ".sculpt_mask"))
+      {
+        return;
+      }
+      break;
+    }
+  }
+
   if (use_pbvh) {
     DRW_shgroup_call_sculpt(pd->sculpt_mask_grp, ob, false, true, true, false, false);
   }