Fix: incorrect indentation in action editor with layered actions

The code that decides the indentation for channel groups and fcurve channels
already accounted for the difference between the Action Editor and Dopesheet/
Graph Editor, since the level of nesting is different in the Action Editor vs
the other two.  However, it didn't account for the difference between legacy and
layered actions, which have different nesting in the Action Editor.

Additionally, before layered actions were introduced it was impossible for
the Action Editor to display channels from e.g. materials, but those channels
can now be displayed in the Action Editor via the slot system.  However, the
indentation code was structured such that material (etc.) channels were *always*
indented as if they were in the Dopesheet/Graph Editor, where they're always
nested under the IDs that use them.

This commit fixes the issue by having a separate code path for indenting in the
Action Editor that only indents channel groups and fcurve channels, and which
properly handles the difference between layered and legacy actions.

Pull Request: https://projects.blender.org/blender/blender/pulls/127202

source/blender/editors/animation/anim_channels_defines.cc

@@ -286,6 +286,36 @@ static short acf_generic_group_offset(bAnimContext *ac, bAnimListElem *ale)
   short offset = acf_generic_basic_offset(ac, ale);
 
   if (ale->id) {
+
+    /* Action Editor. */
+    if (ac->datatype == ANIMCONT_ACTION) {
+      /* For Action Editor mode, we have a limited set of channel types we need
+       * to account for, so we can handle them very simply here in one place. */
+      switch (ale->type) {
+        case ANIMTYPE_FCURVE:
+        case ANIMTYPE_GROUP: {
+          const bAction *action = reinterpret_cast<bAction *>(ale->fcurve_owner_id);
+          if (action->wrap().is_action_layered()) {
+            offset += short(0.35f * U.widget_unit);
+          }
+          break;
+        }
+
+        case ANIMTYPE_SUMMARY:
+        case ANIMTYPE_ACTION_SLOT:
+          break;
+
+        /* There should be no types except the above in Action Editor mode. */
+        default:
+          BLI_assert_unreachable();
+          break;
+      }
+
+      return offset;
+    }
+
+    /* Other editors. */
+
     /* texture animdata */
     if (GS(ale->id->name) == ID_TE) {
       offset += U.widget_unit;
@@ -296,7 +326,7 @@ static short acf_generic_group_offset(bAnimContext *ac, bAnimListElem *ale)
     }
     /* If not in Action Editor mode, action-groups (and their children)
      * must carry some offset too. */
-    else if (ac->datatype != ANIMCONT_ACTION) {
+    else {
       offset += short(0.7f * U.widget_unit);
     }
 

source/blender/editors/animation/anim_filter.cc

@@ -835,6 +835,8 @@ static bAnimListElem *make_new_animlistelem(
       break;
     }
     case ANIMTYPE_GROUP: {
+      BLI_assert_msg(GS(fcurve_owner_id->name) == ID_AC, "fcurve_owner_id should be an Action");
+
       bActionGroup *agrp = (bActionGroup *)data;
 
       ale->flag = agrp->flag;

source/blender/editors/include/ED_anim_api.hh

@@ -304,15 +304,21 @@ struct bAnimListElem {
   Main *bmain;
 
   /**
-   * For list element which corresponds to a f-curve, this is an ID which
-   * owns the f-curve.
+   * For list elements that correspond to an f-curve, a channel group, or an
+   * action slot, this is the ID which owns that data.
    *
-   * For example, if the f-curve is coming from Action, this id will be set to
-   * action's ID. But if this is a f-curve which is a driver, then the owner
-   * is set to, for example, object.
+   * For channel groups and action slots, that will always be an Action. For
+   * f-curves it's more complicated, because f-curves are sometimes owned by
+   * other ID types (e.g. driver f-curves are owned by objects, materials,
+   * etc.), so you have to be careful.
    *
    * NOTE: this is different from id above. The id above will be set to
    * an object if the f-curve is coming from action associated with that object.
+   *
+   * TODO: the responsibilities for this are getting overloaded, which makes it
+   * difficult to use confidently, and also makes its name misleading. Split off
+   * a separate `bAction` pointer that is simply null when the data isn't owned
+   * by an action.
    */
   ID *fcurve_owner_id;