Sculpt: Add Lasso Hide tool

This commit adds the `SCULPT_OT_hide_show_lasso_gesture` and the
corresponding Lasso Hide tool.

* Exposes the selection type for both the lasso and box hide tools
  as a option in the header
* Adds functionality into `sculpt_gesture.cc` for handling lasso
  selections with the `Outside` selection type

For `SelectionType::Outside`, the current implementation opts to not
do any filtering on the PBVH node level due to cases where the node
is mostly covered by a single gesture.

Addresses one of the tools in #80390

Pull Request: https://projects.blender.org/blender/blender/pulls/119140

scripts/presets/keyconfig/keymap_data/blender_default.py

@@ -7892,6 +7892,21 @@ def km_3d_view_tool_sculpt_box_hide(params):
     )
 
 
+def km_3d_view_tool_sculpt_lasso_hide(params):
+    return (
+        "3D View Tool: Sculpt, Lasso Hide",
+        {"space_type": 'VIEW_3D', "region_type": 'WINDOW'},
+        {"items": [
+            ("paint.hide_show_lasso_gesture", params.tool_maybe_tweak_event,
+             {"properties": [("action", 'HIDE')]}),
+            ("paint.hide_show_lasso_gesture", {**params.tool_maybe_tweak_event, "ctrl": True},
+             {"properties": [("action", 'SHOW')]}),
+            ("paint.hide_show_all", {"type": params.select_mouse, "value": params.select_mouse_value},
+             {"properties": [("action", 'SHOW')]}),
+        ]},
+    )
+
+
 def km_3d_view_tool_sculpt_box_mask(params):
     return (
         "3D View Tool: Sculpt, Box Mask",
@@ -8737,6 +8752,7 @@ def generate_keymaps(params=None):
         km_3d_view_tool_edit_curve_extrude_to_cursor(params),
         km_3d_view_tool_edit_curves_draw(params),
         km_3d_view_tool_sculpt_box_hide(params),
+        km_3d_view_tool_sculpt_lasso_hide(params),
         km_3d_view_tool_sculpt_box_mask(params),
         km_3d_view_tool_sculpt_lasso_mask(params),
         km_3d_view_tool_sculpt_box_face_set(params),

scripts/startup/bl_ui/space_toolsystem_toolbar.py

@@ -1381,12 +1381,32 @@ class _defs_sculpt:
 
     @ToolDef.from_fn
     def hide_border():
+        def draw_settings(_context, layout, tool):
+            props = tool.operator_properties("paint.hide_show")
+            layout.prop(props, "area", expand=False)
+
         return dict(
             idname="builtin.box_hide",
             label="Box Hide",
             icon="ops.sculpt.border_hide",
             widget=None,
             keymap=(),
+            draw_settings=draw_settings,
+        )
+
+    @ToolDef.from_fn
+    def hide_lasso():
+        def draw_settings(_context, layout, tool):
+            props = tool.operator_properties("paint.hide_show_lasso_gesture")
+            layout.prop(props, "area", expand=False)
+
+        return dict(
+            idname="builtin.lasso_hide",
+            label="Lasso Hide",
+            icon="ops.sculpt.lasso_hide",
+            widget=None,
+            keymap=(),
+            draw_settings=draw_settings,
         )
 
     @ToolDef.from_fn
@@ -3077,7 +3097,10 @@ class VIEW3D_PT_tools_active(ToolSelectPanelHelper, Panel):
                 _defs_sculpt.mask_lasso,
                 _defs_sculpt.mask_line,
             ),
-            _defs_sculpt.hide_border,
+            (
+                _defs_sculpt.hide_border,
+                _defs_sculpt.hide_lasso
+            ),
             (
                 _defs_sculpt.face_set_box,
                 _defs_sculpt.face_set_lasso,

scripts/startup/bl_ui/space_view3d.py

@@ -3573,6 +3573,12 @@ class VIEW3D_MT_sculpt(Menu):
         props = layout.operator("paint.hide_show", text="Box Show")
         props.action = 'SHOW'
 
+        props = layout.operator("paint.hide_show_lasso_gesture", text="Lasso Hide")
+        props.action = 'HIDE'
+
+        props = layout.operator("paint.hide_show_lasso_gesture", text="Lasso Show")
+        props.action = 'SHOW'
+
         layout.separator()
 
         props = layout.operator("sculpt.face_set_change_visibility", text="Toggle Visibility")

source/blender/editors/datafiles/CMakeLists.txt

@@ -874,6 +874,7 @@ set_property(GLOBAL PROPERTY ICON_GEOM_NAMES
   ops.sculpt.color_filter
   ops.sculpt.face_set_edit
   ops.sculpt.lasso_face_set
+  ops.sculpt.lasso_hide
   ops.sculpt.lasso_mask
   ops.sculpt.lasso_trim
   ops.sculpt.line_mask

source/blender/editors/sculpt_paint/paint_hide.cc

@@ -675,6 +675,17 @@ static int hide_show_gesture_box_exec(bContext *C, wmOperator *op)
   return OPERATOR_FINISHED;
 }
 
+static int hide_show_gesture_lasso_exec(bContext *C, wmOperator *op)
+{
+  std::unique_ptr<gesture::GestureData> gesture_data = gesture::init_from_lasso(C, op);
+  if (!gesture_data) {
+    return OPERATOR_CANCELLED;
+  }
+  hide_show_init_properties(*C, *gesture_data, *op);
+  gesture::apply(*C, *gesture_data, *op);
+  return OPERATOR_FINISHED;
+}
+
 static void hide_show_operator_properties(wmOperatorType *ot)
 {
   static const EnumPropertyItem action_items[] = {
@@ -765,6 +776,26 @@ void PAINT_OT_hide_show(wmOperatorType *ot)
   gesture::operator_properties(ot);
 }
 
+void PAINT_OT_hide_show_lasso_gesture(wmOperatorType *ot)
+{
+  ot->name = "Hide/Show Lasso";
+  ot->idname = "PAINT_OT_hide_show_lasso_gesture";
+  ot->description = "Hide/show some vertices";
+
+  ot->invoke = WM_gesture_lasso_invoke;
+  ot->modal = WM_gesture_lasso_modal;
+  ot->exec = hide_show_gesture_lasso_exec;
+  /* Sculpt-only for now. */
+  ot->poll = SCULPT_mode_poll_view3d;
+
+  ot->flag = OPTYPE_REGISTER | OPTYPE_DEPENDS_ON_CURSOR;
+
+  WM_operator_properties_gesture_lasso(ot);
+  hide_show_operator_properties(ot);
+  hide_show_operator_gesture_properties(ot);
+  gesture::operator_properties(ot);
+}
+
 static void invert_visibility_mesh(Object &object, const Span<PBVHNode *> nodes)
 {
   Mesh &mesh = *static_cast<Mesh *>(object.data);

source/blender/editors/sculpt_paint/paint_intern.hh

@@ -470,6 +470,7 @@ void tag_update_visibility(const bContext &C);
 void PAINT_OT_hide_show_masked(wmOperatorType *ot);
 void PAINT_OT_hide_show_all(wmOperatorType *ot);
 void PAINT_OT_hide_show(wmOperatorType *ot);
+void PAINT_OT_hide_show_lasso_gesture(wmOperatorType *ot);
 
 void PAINT_OT_visibility_invert(wmOperatorType *ot);
 }  // namespace blender::ed::sculpt_paint::hide

source/blender/editors/sculpt_paint/paint_ops.cc

@@ -1550,6 +1550,7 @@ void ED_operatortypes_paint()
   WM_operatortype_append(hide::PAINT_OT_hide_show_all);
   WM_operatortype_append(hide::PAINT_OT_hide_show_masked);
   WM_operatortype_append(hide::PAINT_OT_hide_show);
+  WM_operatortype_append(hide::PAINT_OT_hide_show_lasso_gesture);
   WM_operatortype_append(hide::PAINT_OT_visibility_invert);
 
   /* paint masking */

source/blender/editors/sculpt_paint/sculpt_gesture.cc

@@ -338,6 +338,12 @@ static void update_affected_nodes_by_clip_planes(GestureData &gesture_data)
       case SelectionType::Inside:
         return BKE_pbvh_node_frustum_contain_AABB(&node, &frustum);
       case SelectionType::Outside:
+        /* Certain degenerate cases of a lasso shape can cause the resulting
+         * frustum planes to enclose a node's AABB, therefore we must submit it
+         * to be more throughly evaluated. */
+        if (gesture_data.shape_type == ShapeType::Lasso) {
+          return true;
+        }
         return BKE_pbvh_node_frustum_exclude_AABB(&node, &frustum);
       default:
         BLI_assert_unreachable();
@@ -376,13 +382,21 @@ static bool is_affected_lasso(GestureData &gesture_data, const float co[3])
   /* Clip against lasso boundbox. */
   LassoData *lasso = &gesture_data.lasso;
   if (!BLI_rcti_isect_pt(&lasso->boundbox, scr_co_s[0], scr_co_s[1])) {
-    return false;
+    return gesture_data.selection_type == SelectionType::Outside;
   }
 
   scr_co_s[0] -= lasso->boundbox.xmin;
   scr_co_s[1] -= lasso->boundbox.ymin;
 
-  return lasso->mask_px[scr_co_s[1] * lasso->width + scr_co_s[0]].test();
+  const bool bitmap_result = lasso->mask_px[scr_co_s[1] * lasso->width + scr_co_s[0]].test();
+  switch (gesture_data.selection_type) {
+    case SelectionType::Inside:
+      return bitmap_result;
+    case SelectionType::Outside:
+      return !bitmap_result;
+  }
+  BLI_assert_unreachable();
+  return false;
 }
 
 bool is_affected(GestureData &gesture_data, const float3 &co, const float3 &vertex_normal)

source/blender/windowmanager/intern/wm_operators.cc

@@ -4213,6 +4213,7 @@ static void gesture_lasso_modal_keymap(wmKeyConfig *keyconf)
   WM_modalkeymap_assign(keymap, "GRAPH_OT_select_lasso");
   WM_modalkeymap_assign(keymap, "NODE_OT_select_lasso");
   WM_modalkeymap_assign(keymap, "UV_OT_select_lasso");
+  WM_modalkeymap_assign(keymap, "PAINT_OT_hide_show_lasso_gesture");
 }
 
 /* Zoom to border modal operators. */