Fix #126939: Smooth / Enhance Details brush crash when using tablet

Prior to this commit, the brush action was determined on a step by step
basis by peeking at the `StrokeCache` `bstrength` property and
determining which brush to use based on if the value was negative or
not. The sign of this value, however, was not static across the
entirety of a brush stroke, as it is calculated from three separate
fields, one of which could vary over the course of a stroke when using a
tablet, the `pen_flip` property.

To fix this issue, this commit ensures the `pen_flip` field of the
`StrokeCache` is only updated at the beginning of the stroke. It also
adds a new boolean to store the initial direction of the stroke to
reduce further ambiguity when comparing the sign of the brush strength.

Additionally, the operator level `pen_flip` property is moved to the
generic paint stroke operator instead of being defined as a property of
the `OperatorStrokeElement` struct. This value is now only calculated
at the beginning of the stroke instead of before each stroke step.

Pull Request: https://projects.blender.org/blender/blender/pulls/129559

source/blender/editors/sculpt_paint/paint_stroke.cc

@@ -620,7 +620,6 @@ static void paint_brush_stroke_add_step(
     RNA_float_set_array(&itemptr, "mouse", mouse_out);
     /* Original mouse coordinates. */
     RNA_float_set_array(&itemptr, "mouse_event", mval);
-    RNA_boolean_set(&itemptr, "pen_flip", stroke->pen_flip);
     RNA_float_set(&itemptr, "pressure", pressure);
     RNA_float_set(&itemptr, "x_tilt", stroke->x_tilt);
     RNA_float_set(&itemptr, "y_tilt", stroke->y_tilt);
@@ -1505,6 +1504,8 @@ int paint_stroke_modal(bContext *C, wmOperator *op, const wmEvent *event, PaintS
 
   /* one time stroke initialization */
   if (!stroke->stroke_started) {
+    RNA_boolean_set(op->ptr, "pen_flip", stroke->pen_flip);
+
     stroke->last_pressure = sample_average.pressure;
     stroke->last_mouse_position = sample_average.mouse;
     if (paint_stroke_use_scene_spacing(*br, mode)) {

source/blender/editors/sculpt_paint/paint_utils.cc

@@ -205,6 +205,12 @@ void paint_stroke_operator_properties(wmOperatorType *ot)
                       "Stroke Mode",
                       "Action taken when a paint stroke is made");
   RNA_def_property_flag(prop, PropertyFlag(PROP_SKIP_SAVE));
+
+  /* TODO: Pen flip logic should likely be combined into the stroke mode logic instead of being
+   * an entirely separate concept. */
+  prop = RNA_def_boolean(
+      ot->srna, "pen_flip", false, "Pen Flip", "Whether a tablet's eraser mode is being used");
+  RNA_def_property_flag(prop, PropertyFlag(PROP_HIDDEN | PROP_SKIP_SAVE));
 }
 
 /* 3D Paint */

source/blender/editors/sculpt_paint/sculpt.cc

@@ -2059,6 +2059,23 @@ void calc_area_normal_and_center(const Depsgraph &depsgraph,
 /** \name Generic Brush Utilities
  * \{ */
 
+/**
+ * Calculates the sign of the direction of the brush stroke, typically indicates whether the stroke
+ * will deform a surface inwards or outwards along the brush normal.
+ */
+static float brush_flip(const Brush &brush, const blender::ed::sculpt_paint::StrokeCache &cache)
+{
+  if (brush.flag & BRUSH_INVERT_TO_SCRAPE_FILL) {
+    return 1.0f;
+  }
+
+  const float dir = (brush.flag & BRUSH_DIR_IN) ? -1.0f : 1.0f;
+  const float pen_flip = cache.pen_flip ? -1.0f : 1.0f;
+  const float invert = cache.invert ? -1.0f : 1.0f;
+
+  return dir * pen_flip * invert;
+}
+
 /**
  * Return modified brush strength. Includes the direction of the brush, positive
  * values pull vertices, negative values push. Uses tablet pressure and a
@@ -2076,18 +2093,12 @@ static float brush_strength(const Sculpt &sd,
   /* Primary strength input; square it to make lower values more sensitive. */
   const float root_alpha = BKE_brush_alpha_get(scene, &brush);
   const float alpha = root_alpha * root_alpha;
-  const float dir = (brush.flag & BRUSH_DIR_IN) ? -1.0f : 1.0f;
   const float pressure = BKE_brush_use_alpha_pressure(&brush) ? cache.pressure : 1.0f;
-  const float pen_flip = cache.pen_flip ? -1.0f : 1.0f;
-  const float invert = cache.invert ? -1.0f : 1.0f;
   float overlap = ups.overlap_factor;
   /* Spacing is integer percentage of radius, divide by 50 to get
    * normalized diameter. */
 
-  float flip = dir * invert * pen_flip;
-  if (brush.flag & BRUSH_INVERT_TO_SCRAPE_FILL) {
-    flip = 1.0f;
-  }
+  const float flip = brush_flip(brush, cache);
 
   /* Pressure final value after being tweaked depending on the brush. */
   float final_pressure;
@@ -3169,7 +3180,7 @@ static void do_brush_action(const Depsgraph &depsgraph,
          * stroke strength can become 0 during the stroke, but it can not change sign (the sign is
          * determined in the beginning of the stroke. So here it is important to not switch to
          * enhance brush in the middle of the stroke. */
-        if (ss.cache->bstrength < 0.0f) {
+        if (ss.cache->initial_direction_flipped) {
           /* Invert mode, intensify details. */
           do_enhance_details_brush(depsgraph, sd, ob, node_mask);
         }
@@ -3867,6 +3878,7 @@ static void sculpt_update_cache_invariants(
   copy_v3_v3(cache->initial_normal, ss.cursor_normal);
 
   mode = RNA_enum_get(op->ptr, "mode");
+  cache->pen_flip = RNA_boolean_get(op->ptr, "pen_flip");
   cache->invert = mode == BRUSH_STROKE_INVERT;
   cache->alt_smooth = mode == BRUSH_STROKE_SMOOTH;
   cache->normal_weight = brush->normal_weight;
@@ -3899,6 +3911,8 @@ static void sculpt_update_cache_invariants(
   copy_v2_v2(cache->mouse_event, cache->initial_mouse);
   copy_v2_v2(ups->tex_mouse, cache->initial_mouse);
 
+  cache->initial_direction_flipped = brush_flip(*brush, *cache) < 0.0f;
+
   /* Truly temporary data that isn't stored in properties. */
   cache->vc = vc;
   cache->brush = brush;
@@ -4245,7 +4259,6 @@ static void sculpt_update_cache_variants(bContext *C, Sculpt &sd, Object &ob, Po
     RNA_float_get_array(ptr, "location", cache.location);
   }
 
-  cache.pen_flip = RNA_boolean_get(ptr, "pen_flip");
   RNA_float_get_array(ptr, "mouse", cache.mouse);
   RNA_float_get_array(ptr, "mouse_event", cache.mouse_event);
 

source/blender/editors/sculpt_paint/sculpt_intern.hh

@@ -164,6 +164,15 @@ struct StrokeCache {
   } mirror_modifier_clip;
   float2 initial_mouse;
 
+  /**
+   * Some brushes change behavior drastically depending on the directional value (i.e. the smooth
+   * and enhance details functionality being bound to the Smooth brush).
+   *
+   * Storing the initial direction allows discerning the behavior without checking the sign of the
+   * brush direction at every step, which would have ambiguity at 0.
+   */
+  bool initial_direction_flipped;
+
   /* Variants */
   float radius;
   float radius_squared;

source/blender/makesrna/intern/rna_brush.cc

@@ -3894,7 +3894,6 @@ static void rna_def_brush(BlenderRNA *brna)
  * - 3D location of the brush
  * - 2D mouse location
  * - Tablet pressure
- * - Direction flip
  * - Brush type switch
  * - Time
  */
@@ -3931,10 +3930,6 @@ static void rna_def_operator_stroke_element(BlenderRNA *brna)
   RNA_def_property_range(prop, 0.0f, FLT_MAX);
   RNA_def_property_ui_text(prop, "Brush Size", "Brush size in screen space");
 
-  prop = RNA_def_property(srna, "pen_flip", PROP_BOOLEAN, PROP_NONE);
-  RNA_def_property_flag(prop, PROP_IDPROPERTY);
-  RNA_def_property_ui_text(prop, "Flip", "");
-
   prop = RNA_def_property(srna, "x_tilt", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_flag(prop, PROP_IDPROPERTY);
   RNA_def_property_range(prop, -1.0f, 1.0f);