Geometry Nodes: handle matrix to rotation conversion more gracefully

Previously, this conversion would often result in invalid quaternions or
hit an assert in `normalized_to_quat_fast`. It's not super nice to convert
to euler as an intermediate step performance wise, but it seems to be
the easiest solution for now. Extracting rotations from matrices should
not be done all that often anyway.

Pull Request: https://projects.blender.org/blender/blender/pulls/120568

source/blender/blenkernel/intern/type_conversions.cc

@@ -351,7 +351,7 @@ static ColorGeometry4f byte_color_to_color(const ColorGeometry4b &a)
 
 static math::Quaternion float4x4_to_quaternion(const float4x4 &a)
 {
-  return math::to_quaternion(a);
+  return math::normalized_to_quaternion_safe(math::normalize(a));
 }
 
 static float3 quaternion_to_float3(const math::Quaternion &a)

source/blender/blenlib/BLI_math_matrix.hh

@@ -1221,6 +1221,25 @@ template<typename T>
   return to_quaternion<T>(MatBase<T, 3, 3>(mat));
 }
 
+/**
+ * This is "safe" in the sense that the input matrix may not actually encode a rotation but can
+ * also contain shearing etc.
+ */
+template<typename T>
+[[nodiscard]] inline QuaternionBase<T> normalized_to_quaternion_safe(const MatBase<T, 3, 3> &mat)
+{
+  /* Conversion to quaternion asserts when the matrix contains some kinds of shearing, conversion
+   * to euler does not. */
+  /* TODO: Find a better algorithm that can convert untrusted matrices to quaternions directly. */
+  return to_quaternion(to_euler(mat));
+}
+
+template<typename T>
+[[nodiscard]] inline QuaternionBase<T> normalized_to_quaternion_safe(const MatBase<T, 4, 4> &mat)
+{
+  return to_quaternion(to_euler(mat));
+}
+
 template<bool AllowNegativeScale, typename T, int NumCol, int NumRow>
 [[nodiscard]] inline VecBase<T, 3> to_scale(const MatBase<T, NumCol, NumRow> &mat)
 {

source/blender/blenlib/tests/BLI_math_matrix_test.cc

@@ -597,4 +597,21 @@ TEST(math_matrix, MatrixProjection)
   EXPECT_M4_NEAR(pers2, expect, 1e-5);
 }
 
+TEST(math_matrix, ToQuaternionSafe)
+{
+  float3x3 mat;
+  mat[0] = {0.493316412f, -0.0f, 0.869849861f};
+  mat[1] = {-0.0f, 1.0f, 0.0f};
+  mat[2] = {-0.0176299568f, -0.0f, 0.999844611f};
+
+  float3x3 expect;
+  expect[0] = {0.493316f, 0.000000f, 0.869850f};
+  expect[1] = {-0.000000f, 1.000000f, 0.000000f};
+  expect[2] = {-0.869850f, -0.000000f, 0.493316f};
+
+  /* This is mainly testing if there are any asserts hit because the matrix has shearing. */
+  Quaternion rotation = math::normalized_to_quaternion_safe(normalize(mat));
+  EXPECT_M3_NEAR(from_rotation<float3x3>(rotation), expect, 1e-5);
+}
+
 }  // namespace blender::tests

source/blender/nodes/function/nodes/node_fn_separate_transform.cc

@@ -61,12 +61,14 @@ class SeparateTransformFunction : public mf::MultiFunction {
     }
     else if (!rotation.is_empty() && scale.is_empty()) {
       mask.foreach_index([&](const int64_t i) {
-        rotation[i] = math::to_quaternion(math::normalize(float3x3(transforms[i])));
+        rotation[i] = math::normalized_to_quaternion_safe(math::normalize(transforms[i]));
       });
     }
     else if (!rotation.is_empty() && !scale.is_empty()) {
       mask.foreach_index([&](const int64_t i) {
-        math::to_rot_scale(float3x3(transforms[i]), rotation[i], scale[i]);
+        const float3x3 normalized_mat = math::normalize_and_get_size(float3x3(transforms[i]),
+                                                                     scale[i]);
+        rotation[i] = math::normalized_to_quaternion_safe(normalized_mat);
       });
     }
   }