Fix #146943: VSE crash due to recursive mask rendering

When a strip modifier uses an adjustment layer that is above it
as the mask input, this leads to recursive rendering. Similar to the
fix in !146624, pass SeqRenderState to modifiers as well.

Pull Request: https://projects.blender.org/blender/blender/pulls/147029

source/blender/sequencer/SEQ_modifier.hh

@@ -20,6 +20,7 @@ struct StripModifierData;
 
 namespace blender::seq {
 
+struct SeqRenderState;
 struct StripScreenQuad;
 struct RenderData;
 
@@ -79,6 +80,7 @@ void modifier_free(StripModifierData *smd);
 void modifier_unique_name(Strip *strip, StripModifierData *smd);
 StripModifierData *modifier_find_by_name(Strip *strip, const char *name);
 void modifier_apply_stack(const RenderData *context,
+                          SeqRenderState *state,
                           const Strip *strip,
                           ImBuf *ibuf,
                           int timeline_frame);

source/blender/sequencer/intern/modifiers/modifier.cc

@@ -326,6 +326,7 @@ void apply_and_advance_mask(float4 /*input*/, float4 & /*result*/, const void *&
  * \a timeline_frame is offset by \a fra_offset only in case we are using a real mask.
  */
 static ImBuf *modifier_render_mask_input(const RenderData *context,
+                                         SeqRenderState *state,
                                          int mask_input_type,
                                          Strip *mask_strip,
                                          Mask *mask_id,
@@ -336,8 +337,7 @@ static ImBuf *modifier_render_mask_input(const RenderData *context,
 
   if (mask_input_type == STRIP_MASK_INPUT_STRIP) {
     if (mask_strip) {
-      SeqRenderState state;
-      mask_input = seq_render_strip(context, &state, mask_strip, timeline_frame);
+      mask_input = seq_render_strip(context, state, mask_strip, timeline_frame);
     }
   }
   else if (mask_input_type == STRIP_MASK_INPUT_ID) {
@@ -353,11 +353,17 @@ static ImBuf *modifier_render_mask_input(const RenderData *context,
 
 static ImBuf *modifier_mask_get(StripModifierData *smd,
                                 const RenderData *context,
+                                SeqRenderState *state,
                                 int timeline_frame,
                                 int fra_offset)
 {
-  return modifier_render_mask_input(
-      context, smd->mask_input_type, smd->mask_strip, smd->mask_id, timeline_frame, fra_offset);
+  return modifier_render_mask_input(context,
+                                    state,
+                                    smd->mask_input_type,
+                                    smd->mask_strip,
+                                    smd->mask_id,
+                                    timeline_frame,
+                                    fra_offset);
 }
 
 /* -------------------------------------------------------------------- */
@@ -497,6 +503,7 @@ static bool skip_modifier(Scene *scene, const StripModifierData *smd, int timeli
 }
 
 void modifier_apply_stack(const RenderData *context,
+                          SeqRenderState *state,
                           const Strip *strip,
                           ImBuf *ibuf,
                           int timeline_frame)
@@ -529,7 +536,7 @@ void modifier_apply_stack(const RenderData *context,
         frame_offset = smd->mask_id ? ((Mask *)smd->mask_id)->sfra : 0;
       }
 
-      ImBuf *mask = modifier_mask_get(smd, context, timeline_frame, frame_offset);
+      ImBuf *mask = modifier_mask_get(smd, context, state, timeline_frame, frame_offset);
       smti->apply(context, quad, smd, ibuf, mask);
       if (mask) {
         IMB_freeImBuf(mask);

source/blender/sequencer/intern/render.cc

@@ -621,6 +621,7 @@ static void multiply_ibuf(ImBuf *ibuf, const float fmul, const bool multiply_alp
 }
 
 static ImBuf *input_preprocess(const RenderData *context,
+                               SeqRenderState *state,
                                Strip *strip,
                                float timeline_frame,
                                ImBuf *ibuf,
@@ -691,13 +692,14 @@ static ImBuf *input_preprocess(const RenderData *context,
   }
 
   if (strip->modifiers.first) {
-    modifier_apply_stack(context, strip, preprocessed_ibuf, timeline_frame);
+    modifier_apply_stack(context, state, strip, preprocessed_ibuf, timeline_frame);
   }
 
   return preprocessed_ibuf;
 }
 
 static ImBuf *seq_render_preprocess_ibuf(const RenderData *context,
+                                         SeqRenderState *state,
                                          Strip *strip,
                                          ImBuf *ibuf,
                                          float timeline_frame,
@@ -722,7 +724,7 @@ static ImBuf *seq_render_preprocess_ibuf(const RenderData *context,
   }
 
   if (use_preprocess) {
-    ibuf = input_preprocess(context, strip, timeline_frame, ibuf, is_proxy_image);
+    ibuf = input_preprocess(context, state, strip, timeline_frame, ibuf, is_proxy_image);
   }
 
   return ibuf;
@@ -933,6 +935,7 @@ static ImBuf *create_missing_media_image(const RenderData *context, int width, i
 }
 
 static ImBuf *seq_render_image_strip(const RenderData *context,
+                                     SeqRenderState *state,
                                      Strip *strip,
                                      int timeline_frame,
                                      bool *r_is_proxy_image)
@@ -985,7 +988,7 @@ static ImBuf *seq_render_image_strip(const RenderData *context,
 
       if (view_id != context->view_id) {
         ibufs_arr[view_id] = seq_render_preprocess_ibuf(
-            &localcontext, strip, ibufs_arr[view_id], timeline_frame, true, false);
+            &localcontext, state, strip, ibufs_arr[view_id], timeline_frame, true, false);
       }
     }
 
@@ -1105,6 +1108,7 @@ static ImBuf *seq_render_movie_strip_view(const RenderData *context,
 }
 
 static ImBuf *seq_render_movie_strip(const RenderData *context,
+                                     SeqRenderState *state,
                                      Strip *strip,
                                      float timeline_frame,
                                      bool *r_is_proxy_image)
@@ -1150,7 +1154,7 @@ static ImBuf *seq_render_movie_strip(const RenderData *context,
 
       if (view_id != context->view_id && ibuf_arr[view_id]) {
         ibuf_arr[view_id] = seq_render_preprocess_ibuf(
-            &localcontext, strip, ibuf_arr[view_id], timeline_frame, true, false);
+            &localcontext, state, strip, ibuf_arr[view_id], timeline_frame, true, false);
       }
     }
 
@@ -1693,10 +1697,10 @@ static ImBuf *do_render_strip_uncached(const RenderData *context,
     ibuf = seq_render_effect_strip_impl(context, state, strip, timeline_frame);
   }
   else if (strip->type == STRIP_TYPE_IMAGE) {
-    ibuf = seq_render_image_strip(context, strip, timeline_frame, r_is_proxy_image);
+    ibuf = seq_render_image_strip(context, state, strip, timeline_frame, r_is_proxy_image);
   }
   else if (strip->type == STRIP_TYPE_MOVIE) {
-    ibuf = seq_render_movie_strip(context, strip, timeline_frame, r_is_proxy_image);
+    ibuf = seq_render_movie_strip(context, state, strip, timeline_frame, r_is_proxy_image);
   }
   else if (strip->type == STRIP_TYPE_MOVIECLIP) {
     ibuf = seq_render_movieclip_strip(
@@ -1750,7 +1754,7 @@ ImBuf *seq_render_strip(const RenderData *context,
   if (ibuf) {
     use_preprocess = seq_input_have_to_preprocess(context, strip, timeline_frame);
     ibuf = seq_render_preprocess_ibuf(
-        context, strip, ibuf, timeline_frame, use_preprocess, is_proxy_image);
+        context, state, strip, ibuf, timeline_frame, use_preprocess, is_proxy_image);
     intra_frame_cache_put_preprocessed(context->scene, strip, ibuf);
   }