Fix #130513: Grease Pencil: material/vertex color pinning logic fix

When in grease pencil drawing mode, user could pin material or vertex
color to a specific brush or fill tool, but when switching tools, the
display and the logic of how those pinned material/vertex color are used
is inconsistent, leading to confusion.

The main cause of such problem is that the new GPv3 brush/fill tool is
not respecting pinning status, on top of that, user interface display
logic is also wrong. This PR makes sure that:

- Brush/fill tools respect pinning status.
- Buttons on the tool bar always display correct pinning status.
- Brush cursor will use correct color based on pinning status.

Pull Request: https://projects.blender.org/blender/blender/pulls/130968

scripts/startup/bl_ui/properties_paint_common.py

@@ -1537,15 +1537,17 @@ def brush_basic__draw_color_selector(context, layout, brush, gp_settings):
     if brush.gpencil_tool in {'DRAW', 'FILL'}:
         row.separator(factor=1.0)
         sub_row = row.row(align=True)
-        sub_row.enabled = not gp_settings.pin_draw_mode
-        if gp_settings.pin_draw_mode:
+        pin_draw_mode = gp_settings.pin_draw_mode
+        sub_row.enabled = not pin_draw_mode
+        if pin_draw_mode:
             sub_row.prop_enum(gp_settings, "brush_draw_mode", 'MATERIAL', text="", icon='MATERIAL')
             sub_row.prop_enum(gp_settings, "brush_draw_mode", 'VERTEXCOLOR', text="", icon='VPAINT_HLT')
         else:
             sub_row.prop_enum(settings, "color_mode", 'MATERIAL', text="", icon='MATERIAL')
             sub_row.prop_enum(settings, "color_mode", 'VERTEXCOLOR', text="", icon='VPAINT_HLT')
 
-        show_vertex_color = settings.color_mode == 'VERTEXCOLOR' or gp_settings.brush_draw_mode == 'VERTEXCOLOR'
+        show_vertex_color = ((not pin_draw_mode) and settings.color_mode == 'VERTEXCOLOR') or \
+            (pin_draw_mode and gp_settings.brush_draw_mode == 'VERTEXCOLOR')
 
         if show_vertex_color:
             sub_row = row.row(align=True)

source/blender/editors/sculpt_paint/grease_pencil_fill.cc

@@ -41,6 +41,8 @@
 
 #include "GPU_state.hh"
 
+#include "grease_pencil_intern.hh"
+
 #include <list>
 #include <optional>
 
@@ -650,7 +652,9 @@ static bke::CurvesGeometry boundary_to_curves(const Scene &scene,
         pressure, &brush, brush.gpencil_settings);
   }
 
-  if (scene.toolsettings->gp_paint->mode == GPPAINT_FLAG_USE_VERTEXCOLOR) {
+  const bool use_vertex_color = ed::sculpt_paint::greasepencil::brush_using_vertex_color(
+      scene.toolsettings->gp_paint, &brush);
+  if (use_vertex_color) {
     ColorGeometry4f vertex_color;
     srgb_to_linearrgb_v3_v3(vertex_color, brush.rgb);
     vertex_color.a = brush.gpencil_settings->vertex_factor;

source/blender/editors/sculpt_paint/grease_pencil_intern.hh

@@ -91,6 +91,9 @@ float brush_fill_influence(const Scene &scene,
                            const InputSample &sample,
                            float multi_frame_falloff);
 
+/* Based on pinning status, decide whether to use vertex color or material mode for brush. */
+bool brush_using_vertex_color(const GpPaint *gp_paint, const Brush *brush);
+
 /* True if influence of the brush should be inverted. */
 bool is_brush_inverted(const Brush &brush, BrushStrokeMode stroke_mode);
 

source/blender/editors/sculpt_paint/grease_pencil_paint.cc

@@ -312,7 +312,7 @@ struct PaintOperationExecutor {
     settings_ = brush_->gpencil_settings;
 
     use_settings_random_ = (settings_->flag & GP_BRUSH_GROUP_RANDOM) != 0;
-    use_vertex_color_ = (scene_->toolsettings->gp_paint->mode == GPPAINT_FLAG_USE_VERTEXCOLOR);
+    use_vertex_color_ = brush_using_vertex_color(scene_->toolsettings->gp_paint, brush_);
     if (use_vertex_color_) {
       ColorGeometry4f color_base;
       srgb_to_linearrgb_v3_v3(color_base, brush_->rgb);

source/blender/editors/sculpt_paint/grease_pencil_paint_common.cc

@@ -186,6 +186,16 @@ IndexMask brush_point_influence_mask(const Scene &scene,
   return influence_mask;
 }
 
+bool brush_using_vertex_color(const GpPaint *gp_paint, const Brush *brush)
+{
+  const int brush_draw_mode = brush->gpencil_settings->brush_draw_mode;
+  const bool brush_use_pinned_mode = (brush_draw_mode != GP_BRUSH_MODE_ACTIVE);
+  if (brush_use_pinned_mode) {
+    return (brush_draw_mode == GP_BRUSH_MODE_VERTEXCOLOR);
+  }
+  return (gp_paint->mode == GPPAINT_FLAG_USE_VERTEXCOLOR);
+}
+
 bool is_brush_inverted(const Brush &brush, const BrushStrokeMode stroke_mode)
 {
   /* The basic setting is the brush's setting. During runtime, the user can hold down the Ctrl key

source/blender/editors/sculpt_paint/paint_cursor.cc

@@ -64,6 +64,9 @@
 
 #include "bmesh.hh"
 
+/* Needed for determining tool material/vcolor pinning. */
+#include "grease_pencil_intern.hh"
+
 /* TODOs:
  *
  * Some of the cursor drawing code is doing non-draw stuff
@@ -1590,8 +1593,8 @@ static void grease_pencil_brush_cursor_draw(PaintCursorContext *pcontext)
           (brush->gpencil_brush_type == GPAINT_BRUSH_TYPE_DRAW))
       {
 
-        const bool use_vertex_color = (pcontext->scene->toolsettings->gp_paint->mode ==
-                                       GPPAINT_FLAG_USE_VERTEXCOLOR);
+        const bool use_vertex_color = ed::sculpt_paint::greasepencil::brush_using_vertex_color(
+            pcontext->scene->toolsettings->gp_paint, brush);
         const bool use_vertex_color_stroke = use_vertex_color &&
                                              ELEM(brush->gpencil_settings->vertex_mode,
                                                   GPPAINT_MODE_STROKE,