Fix #146409: nested shader node outputs are ignored

This was supported before, and was accidentally removed in 32c301e3cf. This
patch brings back support for nested shader node outputs. Only rule is that they
must not be in a zone.

Pull Request: https://projects.blender.org/blender/blender/pulls/146983

source/blender/nodes/NOD_node_in_compute_context.hh

@@ -43,7 +43,7 @@ struct NodeInContext {
 };
 
 /**
- * Utility struct to pair a socket with a compute context. This unique identifies a socket in a
+ * Utility struct to pair a socket with a compute context. This uniquely identifies a socket in a
  * node-tree evaluation.
  */
 struct SocketInContext {
@@ -65,6 +65,20 @@ struct SocketInContext {
   BLI_STRUCT_EQUALITY_OPERATORS_2(SocketInContext, context_hash(), socket)
 };
 
+/**
+ * Utility struct to pair a tree with a compute context.
+ */
+struct TreeInContext {
+  const ComputeContext *context = nullptr;
+  const bNodeTree *tree = nullptr;
+
+  uint64_t hash() const;
+  ComputeContextHash context_hash() const;
+  const bNodeTree *operator->() const;
+  const bNodeTree &operator*() const;
+  operator bool() const;
+};
+
 /* -------------------------------------------------------------------- */
 /** \name #NodeInContext Inline Methods
  * \{ */
@@ -142,4 +156,35 @@ inline NodeInContext SocketInContext::owner_node() const
 
 /** \} */
 
+/* -------------------------------------------------------------------- */
+/** \name #TreeInContext Inline Methods
+ * \{ */
+
+inline uint64_t TreeInContext::hash() const
+{
+  return get_default_hash(this->context_hash(), this->tree);
+}
+
+inline ComputeContextHash TreeInContext::context_hash() const
+{
+  return context ? context->hash() : ComputeContextHash{};
+}
+
+inline const bNodeTree *TreeInContext::operator->() const
+{
+  return this->tree;
+}
+
+inline const bNodeTree &TreeInContext::operator*() const
+{
+  return *this->tree;
+}
+
+inline TreeInContext::operator bool() const
+{
+  return this->tree != nullptr;
+}
+
+/** \} */
+
 }  // namespace blender::nodes

source/blender/nodes/intern/shader_nodes_inline.cc

@@ -272,23 +272,24 @@ class ShaderNodesInliner {
     return true;
   }
 
-  Vector<SocketInContext> find_final_output_sockets() const
+  Vector<SocketInContext> find_final_output_sockets()
   {
-    const bke::bNodeTreeZones *zones = src_tree_.zones();
-    if (!zones) {
-      return {};
-    }
+    Vector<TreeInContext> trees;
+    this->find_trees_potentially_containing_shader_outputs_recursive(nullptr, src_tree_, trees);
 
     Vector<SocketInContext> output_sockets;
     auto add_output_type = [&](const char *output_type) {
-      for (const bNode *node : src_tree_.nodes_by_type(output_type)) {
-        const bke::bNodeTreeZone *zone = zones->get_zone_by_node(node->identifier);
-        if (zone) {
-          params_.r_error_messages.append({node, TIP_("Output node must not be in zone")});
-          continue;
-        }
-        for (const bNodeSocket *socket : node->input_sockets()) {
-          output_sockets.append({nullptr, socket});
+      for (const TreeInContext &tree : trees) {
+        const bke::bNodeTreeZones &zones = *tree->zones();
+        for (const bNode *node : tree->nodes_by_type(output_type)) {
+          const bke::bNodeTreeZone *zone = zones.get_zone_by_node(node->identifier);
+          if (zone) {
+            params_.r_error_messages.append({node, TIP_("Output node must not be in zone")});
+            continue;
+          }
+          for (const bNodeSocket *socket : node->input_sockets()) {
+            output_sockets.append({tree.context, socket});
+          }
         }
       }
     };
@@ -316,6 +317,39 @@ class ShaderNodesInliner {
     return output_sockets;
   }
 
+  void find_trees_potentially_containing_shader_outputs_recursive(const ComputeContext *context,
+                                                                  const bNodeTree &tree,
+                                                                  Vector<TreeInContext> &r_trees)
+  {
+    const bke::bNodeTreeZones *zones = src_tree_.zones();
+    if (!zones) {
+      return;
+    }
+    if (tree.has_available_link_cycle()) {
+      return;
+    }
+    r_trees.append({context, &tree});
+    for (const bNode *group_node : tree.group_nodes()) {
+      if (group_node->is_muted()) {
+        continue;
+      }
+      const bNodeTree *group = id_cast<const bNodeTree *>(group_node->id);
+      if (!group || ID_MISSING(&group->id)) {
+        continue;
+      }
+      group->ensure_topology_cache();
+      const bke::bNodeTreeZone *zone = zones->get_zone_by_node(group_node->identifier);
+      if (zone) {
+        /* Node groups in zones are ignored. */
+        continue;
+      }
+      const ComputeContext &group_context = compute_context_cache_.for_group_node(
+          context, group_node->identifier, &tree);
+      this->find_trees_potentially_containing_shader_outputs_recursive(
+          &group_context, *group, r_trees);
+    }
+  }
+
   void handle_socket(const SocketInContext &socket)
   {
     if (!socket->is_available()) {