Refactor: Allow checking sculpt color support without BVH

Certain color-related operators and brushes are only relevant for certain
modes. Historically, we have performed this check by inspecting the BVH
tree type. To avoid this as a hard requirement and avoid needing to
ensure that the tree is built in these scenarios, this commit changes
the `color_supported_check` to inspect the source of truth for dyntopo
(`SculptSession.bm`) and multires (`BKE_sculpt_multires_active`).

Pull Request: https://projects.blender.org/blender/blender/pulls/132086

source/blender/editors/sculpt_paint/sculpt.cc

@@ -5233,20 +5233,18 @@ static void stroke_undo_end(const bContext *C, Brush *brush)
 
 namespace blender::ed::sculpt_paint {
 
-bool color_supported_check(const Object &object, ReportList *reports)
+bool color_supported_check(const Scene &scene, Object &object, ReportList *reports)
 {
-  switch (blender::bke::object::pbvh_get(object)->type()) {
-    case blender::bke::pbvh::Type::Mesh:
-      return true;
-    case blender::bke::pbvh::Type::BMesh:
-      BKE_report(reports, RPT_ERROR, "Not supported in dynamic topology mode");
-      return false;
-    case blender::bke::pbvh::Type::Grids:
-      BKE_report(reports, RPT_ERROR, "Not supported in multiresolution mode");
-      return false;
+  if (const SculptSession &ss = *object.sculpt; ss.bm) {
+    BKE_report(reports, RPT_ERROR, "Not supported in dynamic topology mode");
+    return false;
   }
-  BLI_assert_unreachable();
-  return false;
+  else if (BKE_sculpt_multires_active(&scene, &object)) {
+    BKE_report(reports, RPT_ERROR, "Not supported in multiresolution mode");
+    return false;
+  }
+
+  return true;
 }
 
 static bool stroke_test_start(bContext *C, wmOperator *op, const float mval[2])
@@ -5421,7 +5419,7 @@ static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, const wmEvent
   Brush &brush = *BKE_paint_brush(&sd.paint);
 
   if (SCULPT_brush_type_is_paint(brush.sculpt_brush_type) &&
-      !color_supported_check(ob, op->reports))
+      !color_supported_check(scene, ob, op->reports))
   {
     return OPERATOR_CANCELLED;
   }

source/blender/editors/sculpt_paint/sculpt_filter_color.cc

@@ -478,7 +478,7 @@ static int sculpt_color_filter_init(bContext *C, wmOperator *op)
   }
 
   /* Disable for multires and dyntopo for now */
-  if (!bke::object::pbvh_get(ob) || !color_supported_check(ob, op->reports)) {
+  if (!bke::object::pbvh_get(ob) || !color_supported_check(scene, ob, op->reports)) {
     return OPERATOR_CANCELLED;
   }
 

source/blender/editors/sculpt_paint/sculpt_intern.hh

@@ -395,16 +395,13 @@ bool SCULPT_brush_cursor_poll(bContext *C);
 
 namespace blender::ed::sculpt_paint {
 /**
- * Returns true if sculpt session can handle color attributes
- * (pbvh->type() == bke::pbvh::Type::Mesh).  If false an error
- * message will be shown to the user.  Operators should return
- * OPERATOR_CANCELLED in this case.
+ * Returns true if the current Mesh type can handle color attributes. If false an error message
+ * will be shown to the user.  Operators should return OPERATOR_CANCELLED in this case.
  *
- * NOTE: Does not check if a color attribute actually exists.
- * Calling code must handle this itself; in most cases a call to
- * BKE_sculpt_color_layer_create_if_needed() is sufficient.
+ * NOTE: Does not check if a color attribute actually exists. Calling code must handle this itself;
+ * in most cases a call to BKE_sculpt_color_layer_create_if_needed() is sufficient.
  */
-bool color_supported_check(const Object &object, ReportList *reports);
+bool color_supported_check(const Scene &scene, Object &object, ReportList *reports);
 }  // namespace blender::ed::sculpt_paint
 
 /** \} */

source/blender/editors/sculpt_paint/sculpt_ops.cc

@@ -621,7 +621,7 @@ static int sample_color_invoke(bContext *C, wmOperator *op, const wmEvent * /*ev
   Brush &brush = *BKE_paint_brush(&sd.paint);
   SculptSession &ss = *ob.sculpt;
 
-  if (!color_supported_check(ob, op->reports)) {
+  if (!color_supported_check(scene, ob, op->reports)) {
     return OPERATOR_CANCELLED;
   }
 
@@ -824,7 +824,7 @@ static int mask_by_color_invoke(bContext *C, wmOperator *op, const wmEvent *even
   }
 
   /* Color data is not available in multi-resolution or dynamic topology. */
-  if (!color_supported_check(ob, op->reports)) {
+  if (!color_supported_check(scene, ob, op->reports)) {
     return OPERATOR_CANCELLED;
   }