Geometry Nodes: Filter data-block buttons for tools and modifiers

In the data-block selectors in the node editor and the header,
apply a few rules to increase the separation between tool and
modifier node groups.
1. The modifier accepts non-asset groups or assets marked
   as modifiers (created in the modifier context). Tool node groups
   cannot be used by modifiers.
2. The tool context can only edit node group assets marked as tools
   (created in the tool context).

When switching contexts, the node group is cleared if the state would
be invalid. The tool node group is still stored in the editor's "root node
tree" property. Since it isn't retrieved from the context, the pin button
is useless, so it isn't displayed in the tool mode.

See #101778, #111526.

Pull Request: https://projects.blender.org/blender/blender/pulls/112389

scripts/startup/bl_ui/space_node.py

@@ -47,6 +47,7 @@ class NODE_HT_header(Header):
         # Now expanded via the `ui_type`.
         # layout.prop(snode, "tree_type", text="")
 
+        display_pin = True
         if snode.tree_type == 'ShaderNodeTree':
             layout.prop(snode, "shader_type", text="")
 
@@ -166,6 +167,7 @@ class NODE_HT_header(Header):
                 layout.template_ID(snode, "node_tree", new="node.new_geometry_node_group_tool")
                 if snode.node_tree and snode.node_tree.asset_data:
                     layout.popover(panel="NODE_PT_geometry_node_asset_traits")
+                display_pin = False
         else:
             # Custom node tree is edited as independent ID block
             NODE_MT_editor_menus.draw_collapsible(context, layout)
@@ -175,7 +177,7 @@ class NODE_HT_header(Header):
             layout.template_ID(snode, "node_tree", new="node.new_node_tree")
 
         # Put pin next to ID block
-        if not is_compositor:
+        if not is_compositor and display_pin:
             layout.prop(snode, "pin", text="", emboss=False)
 
         layout.separator_spacer()

source/blender/makesrna/intern/rna_modifier.cc

@@ -1676,7 +1676,20 @@ static bool rna_Modifier_show_expanded_get(PointerRNA *ptr)
 static bool rna_NodesModifier_node_group_poll(PointerRNA * /*ptr*/, PointerRNA value)
 {
   bNodeTree *ntree = static_cast<bNodeTree *>(value.data);
-  return ntree->type == NTREE_GEOMETRY;
+  if (ntree->type != NTREE_GEOMETRY) {
+    return false;
+  }
+  if (ntree->id.asset_data) {
+    if ((ntree->geometry_node_asset_traits->flag & GEO_NODE_ASSET_MODIFIER) == 0) {
+      /* Only node group assets specically marked as modifiers can be modifiers. */
+      return false;
+    }
+  }
+  if (ntree->geometry_node_asset_traits->flag & GEO_NODE_ASSET_TOOL) {
+    /* Tool node groups cannot be modifiers. */
+    return false;
+  }
+  return true;
 }
 
 static void rna_NodesModifier_node_group_update(Main *bmain, Scene *scene, PointerRNA *ptr)

source/blender/makesrna/intern/rna_space.cc

@@ -2509,13 +2509,59 @@ static void rna_SpaceNodeEditor_node_tree_set(PointerRNA *ptr,
   ED_node_tree_start(snode, (bNodeTree *)value.data, nullptr, nullptr);
 }
 
+static bool space_node_node_geometry_nodes_tool_poll(const SpaceNode &snode,
+                                                     const bNodeTree &ntree)
+{
+  if (snode.geometry_nodes_type == SNODE_GEOMETRY_TOOL) {
+    if (!ntree.id.asset_data) {
+      /* Only assets can be tools. */
+      return false;
+    }
+    if ((ntree.geometry_node_asset_traits->flag & GEO_NODE_ASSET_TOOL) == 0) {
+      /* Only node groups specifically marked as tools can be tools. */
+      return false;
+    }
+  }
+  else {
+    if (ntree.geometry_node_asset_traits->flag & GEO_NODE_ASSET_TOOL) {
+      /* Tool node groups cannot be modifiers. */
+      return false;
+    }
+  }
+  return true;
+}
+
 static bool rna_SpaceNodeEditor_node_tree_poll(PointerRNA *ptr, const PointerRNA value)
 {
   SpaceNode *snode = (SpaceNode *)ptr->data;
   bNodeTree *ntree = (bNodeTree *)value.data;
 
   /* node tree type must match the selected type in node editor */
-  return (STREQ(snode->tree_idname, ntree->idname));
+  if (!STREQ(snode->tree_idname, ntree->idname)) {
+    return false;
+  }
+  if (ntree->type == NTREE_GEOMETRY) {
+    if (snode->geometry_nodes_type == SNODE_GEOMETRY_TOOL) {
+      if (!space_node_node_geometry_nodes_tool_poll(*snode, *ntree)) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+static void rna_SpaceNodeEditor_geometry_nodes_type_update(Main * /*bmain*/,
+                                                           Scene * /*scene*/,
+                                                           PointerRNA *ptr)
+{
+  SpaceNode &snode = *static_cast<SpaceNode *>(ptr->data);
+  if (snode.nodetree) {
+    if (snode.nodetree->type == NTREE_GEOMETRY) {
+      if (!space_node_node_geometry_nodes_tool_poll(snode, *snode.nodetree)) {
+        snode.nodetree = nullptr;
+      }
+    }
+  }
 }
 
 static void rna_SpaceNodeEditor_node_tree_update(const bContext *C, PointerRNA * /*ptr*/)
@@ -7450,7 +7496,8 @@ static void rna_def_space_node(BlenderRNA *brna)
   RNA_def_property_enum_items(prop, geometry_nodes_type_items);
   RNA_def_property_ui_text(prop, "Geometry Nodes Type", "");
   RNA_def_property_translation_context(prop, BLT_I18NCONTEXT_ID_ID);
-  RNA_def_property_update(prop, NC_SPACE | ND_SPACE_NODE, nullptr);
+  RNA_def_property_update(
+      prop, NC_SPACE | ND_SPACE_NODE, "rna_SpaceNodeEditor_geometry_nodes_type_update");
 
   prop = RNA_def_property(srna, "id", PROP_POINTER, PROP_NONE);
   RNA_def_property_clear_flag(prop, PROP_EDITABLE);