Refactor: Transform: Unify logic of "orient_axis" and "constraint_axis"

In rotation-based transform operations, the rotation axis can be
determined in two ways:
1. Through "orient_axis" (X, Y or Z)
2. Through "constraint_axis"

When the axis is obtained through the constraint, "orient_axis" is
ignored, and the angle may be negated depending on the view orientation
to match the mouse movement.

However, "orient_axis" never has its angle negated. Since the default
orientation is "View", the Z axis is inverted by default, aligning with
the mouse movement but not with the constraint axis.

This causes problems in the Redo Panel because the constraint fields
are hidden in the Rotation operation, so they need to be unset for the
Axis field to work. However, if you change the value of the Rotation
field, the object may have its rotation negated unexpectedly.

This issue was partially shown in #93078. Commit c30e6a37b0 attempted
to fix it by unsetting the constraint property when the Axis was
changed. However, this solution is incomplete: if the Axis is changed
and then reverted, the negative rotation issue reappears. In addition,
it has not been implemented in all operations.

This commit resolves the issue by reverting c30e6a37b0, aligning the
behavior of "orient_axis" and "constraint_axis", and unsetting
"constraint_axis" in `saveTranform`.

A downside of this solution is that it may break operators invoked from
Python that rely on "orient_axis" as the rotation axis, as the rotation
value now needs to be negated.

Pull Request: https://projects.blender.org/blender/blender/pulls/141101

source/blender/editors/transform/transform.cc

@@ -1778,6 +1778,7 @@ void saveTransform(bContext *C, TransInfo *t, wmOperator *op)
         int orient_axis = constraintModeToIndex(t);
         if (orient_axis != -1) {
           RNA_property_enum_set(op->ptr, prop, orient_axis);
+          t->con.mode &= ~CON_APPLY;
         }
       }
       else {
@@ -2213,4 +2214,15 @@ void transform_final_value_get(const TransInfo *t, float *value, const int value
   memcpy(value, t->values_final, sizeof(float) * value_num);
 }
 
+void view_vector_calc(const TransInfo *t, const float focus[3], float r_vec[3])
+{
+  if (t->persp != RV3D_ORTHO) {
+    sub_v3_v3v3(r_vec, t->viewinv[3], focus);
+  }
+  else {
+    copy_v3_v3(r_vec, t->viewinv[2]);
+  }
+  normalize_v3(r_vec);
+}
+
 }  // namespace blender::ed::transform

source/blender/editors/transform/transform.hh

@@ -232,9 +232,7 @@ enum eTConstraint {
   CON_AXIS1 = 1 << 2,
   CON_AXIS2 = 1 << 3,
   CON_SELECT = 1 << 4,
-  /** Does not reorient vector to face viewport when on. */
-  CON_NOFLIP = 1 << 5,
-  CON_USER = 1 << 6,
+  CON_USER = 1 << 5,
 };
 ENUM_OPERATORS(eTConstraint, CON_USER)
 
@@ -590,8 +588,7 @@ struct TransCon {
   void (*applyRot)(const TransInfo *t,
                    const TransDataContainer *tc,
                    const TransData *td,
-                   float r_axis[3],
-                   float *r_angle);
+                   float r_axis[3]);
 };
 
 struct MouseInput {
@@ -1009,6 +1006,7 @@ wmKeyMap *transform_modal_keymap(wmKeyConfig *keyconf);
  */
 bool transform_apply_matrix(TransInfo *t, float mat[4][4]);
 void transform_final_value_get(const TransInfo *t, float *value, int value_num);
+void view_vector_calc(const TransInfo *t, const float focus[3], float r_vec[3]);
 
 /** \} */
 

source/blender/editors/transform/transform_constraints.cc

@@ -68,17 +68,6 @@ static void projection_matrix_calc(const TransInfo *t, float r_pmtx[3][3])
   mul_m3_m3m3(r_pmtx, t->spacemtx, mat);
 }
 
-static void view_vector_calc(const TransInfo *t, const float focus[3], float r_vec[3])
-{
-  if (t->persp != RV3D_ORTHO) {
-    sub_v3_v3v3(r_vec, t->viewinv[3], focus);
-  }
-  else {
-    copy_v3_v3(r_vec, t->viewinv[2]);
-  }
-  normalize_v3(r_vec);
-}
-
 /* ************************** CONSTRAINTS ************************* */
 #define CONSTRAIN_EPSILON 0.0001f
 
@@ -561,8 +550,7 @@ static void applyObjectConstraintSize(const TransInfo *t,
 
 static void constraints_rotation_impl(const TransInfo *t,
                                       const float axismtx[3][3],
-                                      float r_axis[3],
-                                      float *r_angle)
+                                      float r_axis[3])
 {
   BLI_assert(t->con.mode & CON_APPLY);
   int mode = t->con.mode & (CON_AXIS0 | CON_AXIS1 | CON_AXIS2);
@@ -581,16 +569,6 @@ static void constraints_rotation_impl(const TransInfo *t,
       copy_v3_v3(r_axis, axismtx[2]);
       break;
   }
-  /* Don't flip axis if asked to or if num input. */
-  if (r_angle &&
-      !((mode & CON_NOFLIP) || hasNumInput(&t->num) || (t->flag & T_INPUT_IS_VALUES_FINAL)))
-  {
-    float view_vector[3];
-    view_vector_calc(t, t->center_global, view_vector);
-    if (dot_v3v3(r_axis, view_vector) > 0.0f) {
-      *r_angle = -(*r_angle);
-    }
-  }
 }
 
 /**
@@ -600,20 +578,14 @@ static void constraints_rotation_impl(const TransInfo *t,
  *
  * In the case of single axis constraints, the rotation axis is directly the one constrained to.
  * For planar constraints (2 axis), the rotation axis is the normal of the plane.
- *
- * The following only applies when #CON_NOFLIP is not set.
- * The vector is then modified to always point away from the screen (in global space)
- * This insures that the rotation is always logically following the mouse.
- * (ie: not doing counterclockwise rotations when the mouse moves clockwise).
  */
 static void applyAxisConstraintRot(const TransInfo *t,
                                    const TransDataContainer * /*tc*/,
                                    const TransData *td,
-                                   float r_axis[3],
-                                   float *r_angle)
+                                   float r_axis[3])
 {
   if (!td && t->con.mode & CON_APPLY) {
-    constraints_rotation_impl(t, t->spacemtx, r_axis, r_angle);
+    constraints_rotation_impl(t, t->spacemtx, r_axis);
   }
 }
 
@@ -624,17 +596,11 @@ static void applyAxisConstraintRot(const TransInfo *t,
  *
  * In the case of single axis constraints, the rotation axis is directly the one constrained to.
  * For planar constraints (2 axis), the rotation axis is the normal of the plane.
- *
- * The following only applies when #CON_NOFLIP is not set.
- * The vector is then modified to always point away from the screen (in global space)
- * This insures that the rotation is always logically following the mouse.
- * (ie: not doing counterclockwise rotations when the mouse moves clockwise).
  */
 static void applyObjectConstraintRot(const TransInfo *t,
                                      const TransDataContainer *tc,
                                      const TransData *td,
-                                     float r_axis[3],
-                                     float *r_angle)
+                                     float r_axis[3])
 {
   if (t->con.mode & CON_APPLY) {
     float tmp_axismtx[3][3];
@@ -655,7 +621,7 @@ static void applyObjectConstraintRot(const TransInfo *t,
       axismtx = transform_object_axismtx_get(t, tc, td);
     }
 
-    constraints_rotation_impl(t, axismtx, r_axis, r_angle);
+    constraints_rotation_impl(t, axismtx, r_axis);
   }
 }
 

source/blender/editors/transform/transform_convert_mesh.cc

@@ -1486,6 +1486,15 @@ static void VertsToTransData(TransInfo *t,
 static void createTransEditVerts(bContext * /*C*/, TransInfo *t)
 {
   FOREACH_TRANS_DATA_CONTAINER (t, tc) {
+    if (t->mode == TFM_NORMAL_ROTATION) {
+      /* Avoid freeing the container by creating a dummy TransData. The Rotate Normal mode uses a
+       * custom array and ignores any elements created for the mesh in transData and similar
+       * structures. */
+      tc->data_len = 1;
+      tc->data = MEM_calloc_arrayN<TransData>(tc->data_len, "TransData Dummy");
+      continue;
+    }
+
     TransDataExtension *tx = nullptr;
     BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
     Mesh *mesh = static_cast<Mesh *>(tc->obedit->data);

source/blender/editors/transform/transform_generics.cc

@@ -398,8 +398,8 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
   }
 
   {
-    short orient_types[3];
-    short orient_type_apply = O_DEFAULT;
+    eTOType orient_types[3];
+    eTOType orient_type_apply = O_DEFAULT;
     float custom_matrix[3][3];
 
     int orient_type_scene = V3D_ORIENT_GLOBAL;
@@ -487,9 +487,7 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
       }
     }
     else {
-      if (t->con.mode & CON_APPLY) {
-        orient_type_apply = O_SET;
-      }
+      orient_type_apply = O_SET;
     }
 
     BLI_assert(!ELEM(-1, orient_type_default, orient_type_set));
@@ -498,9 +496,9 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
       orient_type_set = V3D_ORIENT_CUSTOM_MATRIX;
     }
 
-    orient_types[O_DEFAULT] = short(orient_type_default);
-    orient_types[O_SCENE] = short(orient_type_scene);
-    orient_types[O_SET] = short(orient_type_set);
+    orient_types[O_DEFAULT] = eTOType(orient_type_default);
+    orient_types[O_SCENE] = eTOType(orient_type_scene);
+    orient_types[O_SET] = eTOType(orient_type_set);
 
     for (int i = 0; i < 3; i++) {
       /* For efficiency, avoid calculating the same orientation twice. */

source/blender/editors/transform/transform_mode.cc

@@ -1248,6 +1248,23 @@ void transform_mode_default_modal_orientation_set(TransInfo *t, int type)
   }
 }
 
+void transform_mode_rotation_axis_get(const TransInfo *t, float3 &r_axis)
+{
+  if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
+    t->con.applyRot(t, nullptr, nullptr, r_axis);
+  }
+  else {
+    r_axis = t->spacemtx[t->orient_axis];
+  }
+}
+
+bool transform_mode_is_axis_pointing_to_screen(const TransInfo *t, const float3 &axis)
+{
+  float view_vector[3];
+  view_vector_calc(t, t->center_global, view_vector);
+  return dot_v3v3(axis, view_vector) > 0.0f;
+}
+
 /** \} */
 
 }  // namespace blender::ed::transform

source/blender/editors/transform/transform_mode.hh

@@ -106,6 +106,9 @@ void transform_mode_init(TransInfo *t, wmOperator *op, int mode);
  */
 void transform_mode_default_modal_orientation_set(TransInfo *t, int type);
 
+void transform_mode_rotation_axis_get(const TransInfo *t, float3 &r_axis);
+bool transform_mode_is_axis_pointing_to_screen(const TransInfo *t, const float3 &axis);
+
 /* `transform_mode_align.cc` */
 
 extern TransModeInfo TransMode_align;

source/blender/editors/transform/transform_mode_edge_rotate_normal.cc

@@ -68,13 +68,31 @@ static void applyNormalRotation(TransInfo *t)
 {
   char str[UI_MAX_DRAW_STR];
 
-  float axis_final[3];
-  /* Use the negative axis to match the default Z axis of the view matrix. */
-  negate_v3_v3(axis_final, t->spacemtx[t->orient_axis]);
+  float3 axis_final;
+  transform_mode_rotation_axis_get(t, axis_final);
 
-  if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-    t->con.applyRot(t, nullptr, nullptr, axis_final, nullptr);
+  float angle;
+  if (applyNumInput(&t->num, &angle)) {
+    /* Pass. */
   }
+  else {
+    angle = t->values[0] + t->values_modal_offset[0];
+    if (!(t->flag & T_INPUT_IS_VALUES_FINAL) &&
+        transform_mode_is_axis_pointing_to_screen(t, axis_final))
+    {
+      /* Flip rotation direction if axis is pointing to screen. */
+      angle = -angle;
+    }
+    transform_snap_mixed_apply(t, &angle);
+    if (!(transform_snap_is_active(t) && validSnap(t))) {
+      transform_snap_increment(t, &angle);
+    }
+  }
+
+  float mat[3][3];
+  axis_angle_normalized_to_mat3(mat, axis_final, angle);
+
+  t->values_final[0] = angle;
 
   FOREACH_TRANS_DATA_CONTAINER (t, tc) {
     BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
@@ -84,21 +102,8 @@ static void applyNormalRotation(TransInfo *t)
         tc->custom.mode.data);
     BMLoopNorEditData *lnor_ed = lnors_ed_arr->lnor_editdata;
 
-    float axis[3];
-    float mat[3][3];
-    float angle = t->values[0] + t->values_modal_offset[0];
-    copy_v3_v3(axis, axis_final);
-
-    transform_snap_increment(t, &angle);
-
-    transform_snap_mixed_apply(t, &angle);
-
-    applyNumInput(&t->num, &angle);
-
     headerRotation(t, str, sizeof(str), angle);
 
-    axis_angle_normalized_to_mat3(mat, axis, angle);
-
     for (int i = 0; i < lnors_ed_arr->totloop; i++, lnor_ed++) {
       mul_v3_m3v3(lnor_ed->nloc, mat, lnor_ed->niloc);
 
@@ -106,10 +111,12 @@ static void applyNormalRotation(TransInfo *t)
           bm->lnor_spacearr->lspacearr[lnor_ed->loop_index], lnor_ed->nloc, lnor_ed->clnors_data);
     }
 
-    t->values_final[0] = angle;
+    /* Replaces `recalc_data`, thus disregarding partial update, mirror, correct customdata and
+     * other updates. */
+    DEG_id_tag_update(static_cast<ID *>(tc->obedit->data), ID_RECALC_GEOMETRY);
   }
 
-  recalc_data(t);
+  // recalc_data(t);
 
   ED_area_status_text(t->area, str);
 }

source/blender/editors/transform/transform_mode_push_pull.cc

@@ -47,7 +47,7 @@ static void transdata_elem_push_pull(const TransInfo *t,
   if (t->con.applyRot && t->con.mode & CON_APPLY) {
     float axis[3];
     copy_v3_v3(axis, axis_global);
-    t->con.applyRot(t, tc, td, axis, nullptr);
+    t->con.applyRot(t, tc, td, axis);
 
     mul_m3_v3(td->smtx, axis);
     if (is_lock_constraint) {
@@ -95,7 +95,7 @@ static void applyPushPull(TransInfo *t)
   }
 
   if (t->con.applyRot && t->con.mode & CON_APPLY) {
-    t->con.applyRot(t, nullptr, nullptr, axis_global, nullptr);
+    t->con.applyRot(t, nullptr, nullptr, axis_global);
   }
 
   const bool is_lock_constraint = isLockConstraint(t);

source/blender/editors/transform/transform_mode_rotate.cc

@@ -86,7 +86,7 @@ static void transdata_elem_rotate(const TransInfo *t,
   if (t->con.applyRot) {
     copy_v3_v3(axis_buffer, axis);
     axis_final = axis_buffer;
-    t->con.applyRot(t, tc, td, axis_buffer, nullptr);
+    t->con.applyRot(t, tc, td, axis_buffer);
     angle_final = angle * td->factor;
     /* Even though final angle might be identical to orig value,
      * we have to update the rotation matrix in that case... */
@@ -142,7 +142,7 @@ static float RotationBetween(TransInfo *t, const float p1[3], const float p2[3])
   if (t->con.applyRot != nullptr && (t->con.mode & CON_APPLY)) {
     float axis[3];
 
-    t->con.applyRot(t, nullptr, nullptr, axis, nullptr);
+    t->con.applyRot(t, nullptr, nullptr, axis);
 
     angle = -angle_signed_on_axis_v3v3_v3(start, end, axis);
   }
@@ -285,22 +285,23 @@ static bool clip_uv_transform_rotate(const TransInfo *t, float *vec, float *vec_
 
 static void applyRotation(TransInfo *t)
 {
-  float axis_final[3];
-  float final = t->values[0] + t->values_modal_offset[0];
-
-  if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-    t->con.applyRot(t, nullptr, nullptr, axis_final, &final);
-  }
-  else {
-    negate_v3_v3(axis_final, t->spacemtx[t->orient_axis]);
-  }
+  float3 axis_final;
+  transform_mode_rotation_axis_get(t, axis_final);
 
+  float final;
   if (applyNumInput(&t->num, &final)) {
     /* We have to limit the amount of turns to a reasonable number here,
      * to avoid things getting *very* slow, see how applyRotationValue() handles those... */
     final = large_rotation_limit(final);
   }
   else {
+    final = t->values[0] + t->values_modal_offset[0];
+    if (!(t->flag & T_INPUT_IS_VALUES_FINAL) &&
+        transform_mode_is_axis_pointing_to_screen(t, axis_final))
+    {
+      /* Flip rotation direction if axis is pointing to screen. */
+      final = -final;
+    }
     transform_snap_mixed_apply(t, &final);
     if (!(transform_snap_is_active(t) && validSnap(t))) {
       transform_snap_increment(t, &final);
@@ -332,14 +333,9 @@ static void applyRotation(TransInfo *t)
 
 static void applyRotationMatrix(TransInfo *t, float mat_xform[4][4])
 {
-  float axis_final[3];
+  float3 axis_final;
+  transform_mode_rotation_axis_get(t, axis_final);
   const float angle_final = t->values_final[0];
-  if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-    t->con.applyRot(t, nullptr, nullptr, axis_final, nullptr);
-  }
-  else {
-    negate_v3_v3(axis_final, t->spacemtx[t->orient_axis]);
-  }
 
   float mat3[3][3];
   float mat4[4][4];

source/blender/editors/transform/transform_ops.cc

@@ -973,11 +973,6 @@ static void TRANSFORM_OT_trackball(wmOperatorType *ot)
   properties_register(ot, P_PROPORTIONAL | P_MIRROR | P_SNAP | P_GPENCIL_EDIT | P_CENTER);
 }
 
-static void transform_set_orient_axis(Main * /*main*/, Scene * /*scene*/, PointerRNA *ptr)
-{
-  RNA_struct_property_unset(ptr, "constraint_axis");
-}
-
 static void TRANSFORM_OT_rotate(wmOperatorType *ot)
 {
   /* Identifiers. */
@@ -1002,10 +997,6 @@ static void TRANSFORM_OT_rotate(wmOperatorType *ot)
   properties_register(ot,
                       P_ORIENT_AXIS | P_ORIENT_MATRIX | P_CONSTRAINT | P_PROPORTIONAL | P_MIRROR |
                           P_GEO_SNAP | P_GPENCIL_EDIT | P_CENTER);
-
-  if (PropertyRNA *prop = RNA_struct_type_find_property(ot->srna, "orient_axis")) {
-    RNA_def_property_update_runtime(prop, transform_set_orient_axis);
-  }
 }
 
 static bool tilt_poll(bContext *C)
@@ -1420,6 +1411,7 @@ static void TRANSFORM_OT_rotate_normal(wmOperatorType *ot)
   ot->modal = transform_modal;
   ot->cancel = transform_cancel;
   ot->poll = ED_operator_editmesh;
+  ot->poll_property = transform_poll_property;
 
   RNA_def_float_rotation(
       ot->srna, "value", 0, nullptr, -FLT_MAX, FLT_MAX, "Angle", "", -M_PI * 2, M_PI * 2);