Realtime Compositor: Support realization of transformations

This patch adds support for the realization of transformations of
operation inputs in the Realtime Compositor. Input socket declarations
can now include a preference to what sort of realization needs to
happen.

All inputs specify realization on the operation domain by default
because that is needed for the correct operation of most operations.

Nodes may choose not to be realized on the operation domain, like the
MapUV, Plane Deform, and Bokeh Blur nodes; that's because their inputs
are treated as transform-less image objects.

Nodes may chose to realize their rotation or scale, like operations that
are not rotation or scale invariant and thus need images of identity
transformations. No nodes are declared as such so far, as this is still
being considered by developers and test builds be published for testing.

This patch coincidentally also fixes #102252 by declaring the Bokeh input
of the Bokeh Blur node to need realization of rotation. Which is the only
functional change of the patch.

Fixes #102252.

Pull Request: https://projects.blender.org/blender/blender/pulls/111179

source/blender/compositor/realtime_compositor/COM_input_descriptor.hh

@@ -8,6 +8,24 @@
 
 namespace blender::realtime_compositor {
 
+/* ------------------------------------------------------------------------------------------------
+ * Input Realization Options
+ *
+ * A bit-field that specifies how the input should be realized before execution. See the discussion
+ * in COM_domain.hh for more information on what realization mean. */
+struct InputRealizationOptions {
+  /* The input should be realized on the operation domain of the operation. */
+  bool realize_on_operation_domain : 1;
+  /* The input should be realized on a domain that is identical to the domain of the input but with
+   * an identity rotation and an increased size that completely fits the image after rotation. This
+   * is useful for operations that are not rotation invariant. */
+  bool realize_rotation : 1;
+  /* The input should be realized on a domain that is identical to the domain of the input but with
+   * an identity scale and an increased/decreased size that completely fits the image after
+   * scaling. This is useful for operations that are not scale invariant. */
+  bool realize_scale : 1;
+};
+
 /* ------------------------------------------------------------------------------------------------
  * Input Descriptor
  *
@@ -18,16 +36,15 @@ class InputDescriptor {
    * receive for the input, in which case, an implicit conversion operation will be added as an
    * input processor to convert it to the required type. */
   ResultType type;
-  /* If true, then the input does not need to be realized on the domain of the operation before its
-   * execution. See the discussion in COM_domain.hh for more information. */
-  bool skip_realization = false;
+  /* The options that specify how the input should be realized. */
+  InputRealizationOptions realization_options = {true};
   /* The priority of the input for determining the operation domain. The non-single value input
    * with the highest priority will be used to infer the operation domain, the highest priority
    * being zero. See the discussion in COM_domain.hh for more information. */
   int domain_priority = 0;
   /* If true, the input expects a single value, and if a non-single value is provided, a default
    * single value will be used instead, see the get_<type>_value_default methods in the Result
-   * class. It follows that this also implies skip_realization, because we don't need to realize a
+   * class. It follows that this also implies no realization, because we don't need to realize a
    * result that will be discarded anyways. If false, the input can work with both single and
    * non-single values. */
   bool expects_single_value = false;

source/blender/compositor/realtime_compositor/COM_realize_on_domain_operation.hh

@@ -48,4 +48,34 @@ class RealizeOnDomainOperation : public SimpleOperation {
   GPUShader *get_realization_shader();
 };
 
+/* ------------------------------------------------------------------------------------------------
+ * Realize Transformation Operation
+ *
+ * A simple operation that realizes its input on a domain such that its transformations become
+ * identity and its size is increased/decreased to adapt to the new domain. The transformations
+ * that become identity are the ones marked to be realized in the given InputRealizationOptions.
+ * For instance, if InputRealizationOptions.realize_rotation is true and the input is rotated, the
+ * output of the operation will be a result of zero rotation but expanded size to account for the
+ * bounding box of the input after rotation. This is useful for operations that are not rotation
+ * invariant and thus require an input of zero rotation for correct operation.
+ *
+ * Notice that this class is not an actual operation, but constructs a RealizeOnDomainOperation
+ * with the appreciate domain in its construct_if_needed static constructor. */
+class RealizeTransformationOperation {
+ public:
+  /* Determine if a realize transformation operation is needed for the input with the given result
+   * and descriptor. If it is not needed, return a null pointer. If it is needed, return an
+   * instance of RealizeOnDomainOperation with the appropriate domain. */
+  static SimpleOperation *construct_if_needed(Context &context,
+                                              const Result &input_result,
+                                              const InputDescriptor &input_descriptor);
+
+ private:
+  /* Given the domain of an input and its realization options, compute a domain such that the
+   * appropriate transformations specified in the realization options become identity and the size
+   * of the domain is increased/reduced to adapt to the new domain. */
+  static Domain compute_target_domain(const Domain &input_domain,
+                                      const InputRealizationOptions &realization_options);
+};
+
 }  // namespace blender::realtime_compositor

source/blender/compositor/realtime_compositor/intern/compile_state.cc

@@ -150,8 +150,8 @@ Domain CompileState::compute_shader_node_domain(DNode node)
       continue;
     }
 
-    /* An input that skips realization can't be a domain input. */
-    if (input_descriptor.skip_realization) {
+    /* An input that skips operation domain realization can't be a domain input. */
+    if (!input_descriptor.realization_options.realize_on_operation_domain) {
       continue;
     }
 

source/blender/compositor/realtime_compositor/intern/operation.cc

@@ -70,8 +70,8 @@ Domain Operation::compute_domain()
       continue;
     }
 
-    /* An input that skips realization can't be a domain input. */
-    if (descriptor.skip_realization) {
+    /* An input that skips operation domain realization can't be a domain input. */
+    if (!descriptor.realization_options.realize_on_operation_domain) {
       continue;
     }
 
@@ -106,6 +106,12 @@ void Operation::add_and_evaluate_input_processors()
     add_and_evaluate_input_processor(identifier, conversion);
   }
 
+  for (const StringRef &identifier : results_mapped_to_inputs_.keys()) {
+    SimpleOperation *realize_transformation = RealizeTransformationOperation::construct_if_needed(
+        context(), get_input(identifier), get_input_descriptor(identifier));
+    add_and_evaluate_input_processor(identifier, realize_transformation);
+  }
+
   for (const StringRef &identifier : results_mapped_to_inputs_.keys()) {
     SimpleOperation *realize_on_domain = RealizeOnDomainOperation::construct_if_needed(
         context(), get_input(identifier), get_input_descriptor(identifier), compute_domain());

source/blender/compositor/realtime_compositor/intern/realize_on_domain_operation.cc

@@ -2,7 +2,10 @@
  *
  * SPDX-License-Identifier: GPL-2.0-or-later */
 
+#include "BLI_math_angle_types.hh"
 #include "BLI_math_matrix.hh"
+#include "BLI_math_matrix_types.hh"
+#include "BLI_math_vector_types.hh"
 #include "BLI_utildefines.h"
 
 #include "GPU_shader.h"
@@ -17,6 +20,10 @@
 
 namespace blender::realtime_compositor {
 
+/* ------------------------------------------------------------------------------------------------
+ * Realize On Domain Operation
+ */
+
 RealizeOnDomainOperation::RealizeOnDomainOperation(Context &context,
                                                    Domain domain,
                                                    ResultType type)
@@ -119,8 +126,8 @@ SimpleOperation *RealizeOnDomainOperation::construct_if_needed(
     const InputDescriptor &input_descriptor,
     const Domain &operation_domain)
 {
-  /* This input wants to skip realization, the operation is not needed. */
-  if (input_descriptor.skip_realization) {
+  /* This input doesn't need realization, the operation is not needed. */
+  if (!input_descriptor.realization_options.realize_on_operation_domain) {
     return nullptr;
   }
 
@@ -145,4 +152,68 @@ SimpleOperation *RealizeOnDomainOperation::construct_if_needed(
   return new RealizeOnDomainOperation(context, operation_domain, input_descriptor.type);
 }
 
+/* ------------------------------------------------------------------------------------------------
+ * Realize Transformation Operation
+ */
+
+Domain RealizeTransformationOperation::compute_target_domain(
+    const Domain &input_domain, const InputRealizationOptions &realization_options)
+{
+  if (!realization_options.realize_rotation && !realization_options.realize_scale) {
+    return input_domain;
+  }
+
+  math::AngleRadian rotation;
+  float2 translation, scale;
+  float2 size = float2(input_domain.size);
+  math::to_loc_rot_scale(input_domain.transformation, translation, rotation, scale);
+
+  /* Set the rotation to zero and expand the domain size to fit the bounding box of the rotated
+   * result. */
+  if (realization_options.realize_rotation) {
+    const float sine = math::abs(math::sin(rotation));
+    const float cosine = math::abs(math::cos(rotation));
+    size = float2(size.x * sine + size.y * cosine, size.x * cosine + size.y * sine);
+    rotation = 0.0f;
+  }
+
+  /* Set the scale to 1 and scale the domain size to adapt to the new domain. */
+  if (realization_options.realize_scale) {
+    size *= scale;
+    scale = float2(1.0f);
+  }
+
+  const float3x3 transformation = math::from_loc_rot_scale<float3x3>(translation, rotation, scale);
+
+  return Domain(int2(math::ceil(size)), transformation);
+}
+
+SimpleOperation *RealizeTransformationOperation::construct_if_needed(
+    Context &context, const Result &input_result, const InputDescriptor &input_descriptor)
+{
+  /* The input expects a single value and if no single value is provided, it will be ignored and a
+   * default value will be used, so no need to realize it and the operation is not needed. */
+  if (input_descriptor.expects_single_value) {
+    return nullptr;
+  }
+
+  /* Input result is a single value and does not need realization, the operation is not needed. */
+  if (input_result.is_single_value()) {
+    return nullptr;
+  }
+
+  const Domain target_domain = compute_target_domain(input_result.domain(),
+                                                     input_descriptor.realization_options);
+
+  /* The input have an identical domain to the target domain, either because the input doesn't need
+   * to realize its transformations or because it has identity transformations, so no need to
+   * realize it and the operation is not needed. */
+  if (target_domain == input_result.domain()) {
+    return nullptr;
+  }
+
+  /* Otherwise, realization on the target domain is needed. */
+  return new RealizeOnDomainOperation(context, target_domain, input_descriptor.type);
+}
+
 }  // namespace blender::realtime_compositor

source/blender/compositor/realtime_compositor/intern/utilities.cc

@@ -124,8 +124,18 @@ InputDescriptor input_descriptor_from_input_socket(const bNodeSocket *socket)
   }
   const SocketDeclarationPtr &socket_declaration = node_declaration->inputs[socket->index()];
   input_descriptor.domain_priority = socket_declaration->compositor_domain_priority();
-  input_descriptor.skip_realization = socket_declaration->compositor_skip_realization();
   input_descriptor.expects_single_value = socket_declaration->compositor_expects_single_value();
+
+  input_descriptor.realization_options.realize_on_operation_domain = bool(
+      socket_declaration->compositor_realization_options() &
+      CompositorInputRealizationOptions::RealizeOnOperationDomain);
+  input_descriptor.realization_options.realize_rotation = bool(
+      socket_declaration->compositor_realization_options() &
+      CompositorInputRealizationOptions::RealizeRotation);
+  input_descriptor.realization_options.realize_scale = bool(
+      socket_declaration->compositor_realization_options() &
+      CompositorInputRealizationOptions::RealizeScale);
+
   return input_descriptor;
 }
 

source/blender/nodes/NOD_node_declaration.hh

@@ -4,10 +4,12 @@
 
 #pragma once
 
+#include <cstdint>
 #include <functional>
 #include <type_traits>
 
 #include "BLI_string_ref.hh"
+#include "BLI_utildefines.h"
 #include "BLI_vector.hh"
 
 #include "BLT_translation.h"
@@ -41,6 +43,17 @@ enum class OutputSocketFieldType {
   PartiallyDependent,
 };
 
+/**
+ * A bit-field that maps to the realtime_compositor::InputRealizationOptions.
+ */
+enum class CompositorInputRealizationOptions : uint8_t {
+  None = 0,
+  RealizeOnOperationDomain = (1 << 0),
+  RealizeRotation = (1 << 1),
+  RealizeScale = (1 << 2),
+};
+ENUM_OPERATORS(CompositorInputRealizationOptions, CompositorInputRealizationOptions::RealizeScale)
+
 /**
  * Contains information about how a node output's field state depends on inputs of the same node.
  */
@@ -166,14 +179,13 @@ class SocketDeclaration {
   OutputFieldDependency output_field_dependency;
 
  private:
+  CompositorInputRealizationOptions compositor_realization_options_ =
+      CompositorInputRealizationOptions::RealizeOnOperationDomain;
+
   /** The priority of the input for determining the domain of the node. See
    * realtime_compositor::InputDescriptor for more information. */
   int compositor_domain_priority_ = 0;
 
-  /** This input shouldn't be realized on the operation domain of the node. See
-   * realtime_compositor::InputDescriptor for more information. */
-  bool compositor_skip_realization_ = false;
-
   /** This input expects a single value and can't operate on non-single values. See
    * realtime_compositor::InputDescriptor for more information. */
   bool compositor_expects_single_value_ = false;
@@ -208,8 +220,8 @@ class SocketDeclaration {
    */
   void make_available(bNode &node) const;
 
+  const CompositorInputRealizationOptions &compositor_realization_options() const;
   int compositor_domain_priority() const;
-  bool compositor_skip_realization() const;
   bool compositor_expects_single_value() const;
 
   const ImplicitInputValueFn *implicit_input_fn() const
@@ -424,6 +436,12 @@ class SocketDeclarationBuilder : public BaseSocketDeclarationBuilder {
     return *(Self *)this;
   }
 
+  Self &compositor_realization_options(CompositorInputRealizationOptions value)
+  {
+    decl_->compositor_realization_options_ = value;
+    return *(Self *)this;
+  }
+
   /** The priority of the input for determining the domain of the node. See
    * realtime_compositor::InputDescriptor for more information. */
   Self &compositor_domain_priority(int priority)
@@ -432,14 +450,6 @@ class SocketDeclarationBuilder : public BaseSocketDeclarationBuilder {
     return *(Self *)this;
   }
 
-  /** This input shouldn't be realized on the operation domain of the node. See
-   * realtime_compositor::InputDescriptor for more information. */
-  Self &compositor_skip_realization(bool value = true)
-  {
-    decl_->compositor_skip_realization_ = value;
-    return *(Self *)this;
-  }
-
   /** This input expects a single value and can't operate on non-single values. See
    * realtime_compositor::InputDescriptor for more information. */
   Self &compositor_expects_single_value(bool value = true)
@@ -666,16 +676,17 @@ inline bool operator!=(const FieldInferencingInterface &a, const FieldInferencin
 /** \name #SocketDeclaration Inline Methods
  * \{ */
 
+inline const CompositorInputRealizationOptions &SocketDeclaration::compositor_realization_options()
+    const
+{
+  return compositor_realization_options_;
+}
+
 inline int SocketDeclaration::compositor_domain_priority() const
 {
   return compositor_domain_priority_;
 }
 
-inline bool SocketDeclaration::compositor_skip_realization() const
-{
-  return compositor_skip_realization_;
-}
-
 inline bool SocketDeclaration::compositor_expects_single_value() const
 {
   return compositor_expects_single_value_;

source/blender/nodes/composite/nodes/node_composite_bokehblur.cc

@@ -30,7 +30,7 @@ static void cmp_node_bokehblur_declare(NodeDeclarationBuilder &b)
       .compositor_domain_priority(0);
   b.add_input<decl::Color>("Bokeh")
       .default_value({1.0f, 1.0f, 1.0f, 1.0f})
-      .compositor_skip_realization();
+      .compositor_realization_options(CompositorInputRealizationOptions::RealizeRotation);
   b.add_input<decl::Float>("Size")
       .default_value(1.0f)
       .min(0.0f)

source/blender/nodes/composite/nodes/node_composite_map_uv.cc

@@ -25,7 +25,7 @@ static void cmp_node_map_uv_declare(NodeDeclarationBuilder &b)
 {
   b.add_input<decl::Color>("Image")
       .default_value({1.0f, 1.0f, 1.0f, 1.0f})
-      .compositor_skip_realization();
+      .compositor_realization_options(CompositorInputRealizationOptions::None);
   b.add_input<decl::Vector>("UV")
       .default_value({1.0f, 0.0f, 0.0f})
       .min(0.0f)

source/blender/nodes/composite/nodes/node_composite_planetrackdeform.cc

@@ -40,7 +40,8 @@ NODE_STORAGE_FUNCS(NodePlaneTrackDeformData)
 
 static void cmp_node_planetrackdeform_declare(NodeDeclarationBuilder &b)
 {
-  b.add_input<decl::Color>("Image").compositor_skip_realization();
+  b.add_input<decl::Color>("Image").compositor_realization_options(
+      CompositorInputRealizationOptions::None);
   b.add_output<decl::Color>("Image");
   b.add_output<decl::Float>("Plane");
 }