Fix #134073: Compositor crash with viewer inside muted group

The compositor crashes when the user goes into a muted group that has a
viewer node while the backdrop is enabled. The compositor should not
schedule viewer nodes inside muted contexts, so we need to add checks to
prevent this.

Pull Request: https://projects.blender.org/blender/blender/pulls/134093

source/blender/compositor/intern/scheduler.cc

@@ -21,6 +21,23 @@ namespace blender::compositor {
 
 using namespace nodes::derived_node_tree_types;
 
+/* Returns true if any of the node group nodes that make up this tree context are muted. */
+static bool is_tree_context_muted(const DTreeContext &tree_context)
+{
+  /* Root contexts are never muted. */
+  if (tree_context.is_root()) {
+    return false;
+  }
+
+  /* The node group that represents this context is muted. */
+  if (tree_context.parent_node()->is_muted()) {
+    return true;
+  }
+
+  /* Recursively check parent contexts up until the root context. */
+  return is_tree_context_muted(*tree_context.parent_context());
+}
+
 /* Add the active viewer node in the given tree context to the given stack. If viewer nodes are
  * treated as composite outputs, this function will also add either the viewer or the composite
  * node since composite nodes were skipped in add_output_nodes such that viewer nodes take
@@ -29,6 +46,11 @@ static bool add_viewer_nodes_in_context(const Context &context,
                                         const DTreeContext *tree_context,
                                         Stack<DNode> &node_stack)
 {
+  /* Do not add viewers that are inside muted contexts. */
+  if (is_tree_context_muted(*tree_context)) {
+    return false;
+  }
+
   for (const bNode *node : tree_context->btree().nodes_by_type("CompositorNodeViewer")) {
     if (node->flag & NODE_DO_OUTPUT && !node->is_muted()) {
       node_stack.push(DNode(tree_context, node));