animation retarget, kinda

CUE4Parse

@@ -1 +1 @@
-Subproject commit c7fed92ddb2dc2aaec428504396945deefb1ff22
+Subproject commit 91741c40ca8545c7ea3730493c58475ee93ee465

FModel/MainWindow.xaml.cs

@@ -77,13 +77,13 @@ public partial class MainWindow
 #if DEBUG
         await _threadWorkerView.Begin(cancellationToken =>
             _applicationView.CUE4Parse.Extract(cancellationToken,
-                "MoonMan/Content/DeliverUsTheMoon/Characters/Astronaut/SK_Astronaut.uasset"));
+                "fortnitegame/Content/Characters/Player/Male/Large/Bodies/M_LRG_BasilStrong/Meshes/M_LRG_BasilStrong.uasset"));
         await _threadWorkerView.Begin(cancellationToken =>
             _applicationView.CUE4Parse.Extract(cancellationToken,
-                "MoonMan/Content/DeliverUsTheMoon/Characters/Astronaut/cinematic/A_Astro_Space_Breach_Grab2_Success.uasset"));
+                "fortnitegame/Content/Animation/Game/MainPlayer/Emotes/Alliteration/Emote_Alliteration_CMM.uasset"));
-        await _threadWorkerView.Begin(cancellationToken =>
+        // await _threadWorkerView.Begin(cancellationToken =>
-            _applicationView.CUE4Parse.Extract(cancellationToken,
+        //     _applicationView.CUE4Parse.Extract(cancellationToken,
-                "MoonMan/Content/DeliverUsTheMoon/Characters/Astronaut/AM_OxygenHub_Enter.uasset"));
+        //         "MoonMan/Content/DeliverUsTheMoon/Characters/Astronaut/AM_OxygenHub_Enter.uasset"));
 #endif
     }
 

FModel/Views/Snooper/Models/Animations/Animation.cs

@@ -3,6 +3,7 @@ using System.Numerics;
 using CUE4Parse_Conversion.Animations;
 using CUE4Parse.Utils;
 using ImGuiNET;
+using Serilog;
 
 namespace FModel.Views.Snooper.Models.Animations;
 
@@ -27,6 +28,8 @@ public class Animation : IDisposable
     public readonly Sequence[] Sequences;
     public int SequencesCount => Sequences.Length;
 
+    public readonly Matrix4x4[] InvertedBonesMatrix;
+
     public Animation()
     {
         Reset();
@@ -35,14 +38,31 @@ public class Animation : IDisposable
         EndTime = 0.0f;
         TotalElapsedTime = 0.0f;
         Sequences = Array.Empty<Sequence>();
+        InvertedBonesMatrix = Array.Empty<Matrix4x4>();
     }
 
     public Animation(Skeleton skeleton, CAnimSet anim, bool rotationOnly) : this()
     {
+        InvertedBonesMatrix = new Matrix4x4[skeleton.BoneCount];
+        for (int boneIndex = 0; boneIndex < InvertedBonesMatrix.Length; boneIndex++)
+        {
+            Matrix4x4.Invert(skeleton.BonesTransformByIndex[boneIndex].Matrix, out var inverted);
+            InvertedBonesMatrix[boneIndex] = inverted;
+        }
+
+#if DEBUG
+        for (int trackIndex = 0; trackIndex < anim.TrackBonesInfo.Length; trackIndex++)
+        {
+            var bone = anim.TrackBonesInfo[trackIndex];
+            if (!skeleton.BonesIndicesByLoweredName.TryGetValue(bone.Name.Text.ToLower(), out _))
+                Log.Warning($"Bone Mismatch: {bone.Name.Text} ({trackIndex}) is not present in the mesh's reference skeleton");
+        }
+#endif
+
         Sequences = new Sequence[anim.Sequences.Count];
         for (int i = 0; i < Sequences.Length; i++)
         {
-            Sequences[i] = new Sequence(anim.Sequences[i], skeleton, rotationOnly);
+            Sequences[i] = new Sequence(skeleton, anim, anim.Sequences[i], rotationOnly);
 
             TotalElapsedTime += anim.Sequences[i].NumFrames * Sequences[i].TimePerFrame;
             EndTime = Sequences[i].EndTime;
@@ -63,7 +83,8 @@ public class Animation : IDisposable
     public Matrix4x4 InterpolateBoneTransform(int boneIndex)
     {
         // interpolate here
-        return Sequences[CurrentSequence].BonesTransform[boneIndex][FrameInSequence].Matrix;
+        return InvertedBonesMatrix[boneIndex] *
+               Sequences[CurrentSequence].BonesTransform[boneIndex][FrameInSequence].Matrix;
     }
 
     private void TimeCalculation()
@@ -82,7 +103,7 @@ public class Animation : IDisposable
         for (int s = 0; s < CurrentSequence; s++)
             lastEndTime = Sequences[s].EndTime;
 
-        FrameInSequence = Math.Min(((ElapsedTime - lastEndTime) / Sequences[CurrentSequence].TimePerFrame).FloorToInt(), Sequences[CurrentSequence].UsableEndFrame);
+        FrameInSequence = Math.Min(((ElapsedTime - lastEndTime) / Sequences[CurrentSequence].TimePerFrame).FloorToInt(), Sequences[CurrentSequence].EndFrame);
     }
 
     private void Reset()
@@ -92,6 +113,15 @@ public class Animation : IDisposable
         CurrentSequence = 0;
     }
 
+    public void Dispose()
+    {
+        Reset();
+        for (int i = 0; i < Sequences.Length; i++)
+        {
+            Sequences[i]?.Dispose();
+        }
+    }
+
     private float _timeHeight = 10.0f;
     private float _timeBarHeight => _timeHeight * 2.0f;
     public void ImGuiTimeline(ImFontPtr fontPtr)
@@ -171,9 +201,4 @@ public class Animation : IDisposable
                 throw new ArgumentOutOfRangeException(nameof(separatorType), separatorType, null);
         }
     }
-
-    public void Dispose()
-    {
-
-    }
 }

FModel/Views/Snooper/Models/Animations/Sequence.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Numerics;
 using CUE4Parse_Conversion.Animations;
+using CUE4Parse.UE4.Assets.Exports.Animation;
+using CUE4Parse.UE4.Objects.Core.Math;
 using CUE4Parse.Utils;
 using ImGuiNET;
 
@@ -9,7 +11,6 @@ namespace FModel.Views.Snooper.Models.Animations;
 public class Sequence : IDisposable
 {
     public readonly string Name;
-    public readonly int MaxFrame;
     public readonly float TimePerFrame;
     public readonly float StartTime;
     public readonly float Duration;
@@ -17,25 +18,35 @@ public class Sequence : IDisposable
     public readonly int EndFrame;
     public readonly int LoopingCount;
 
-    public int UsableEndFrame => EndFrame - 1;
-
     public readonly Transform[][] BonesTransform;
 
-    public Sequence(CAnimSequence sequence, Skeleton skeleton, bool rotationOnly)
+    private Sequence(CAnimSequence sequence)
     {
         Name = sequence.Name;
-        MaxFrame = sequence.NumFrames - 1;
         TimePerFrame = 1.0f / sequence.Rate;
         StartTime = sequence.StartPos;
         Duration = sequence.AnimEndTime;
         EndTime = StartTime + Duration;
-        EndFrame = (Duration / TimePerFrame).FloorToInt();
+        EndFrame = (Duration / TimePerFrame).FloorToInt() - 1;
         LoopingCount = sequence.LoopingCount;
+    }
 
-        BonesTransform = new Transform[skeleton.BonesTransformByIndex.Count][];
+    public Sequence(Skeleton skeleton, CAnimSet anim, CAnimSequence sequence, bool rotationOnly) : this(sequence)
-        for (int trackIndex = 0; trackIndex < skeleton.UnrealSkeleton.ReferenceSkeleton.FinalRefBoneInfo.Length; trackIndex++)
+    {
+        BonesTransform = new Transform[skeleton.BoneCount][];
+        for (int boneIndex = 0; boneIndex < BonesTransform.Length; boneIndex++)
         {
-            var bone = skeleton.UnrealSkeleton.ReferenceSkeleton.FinalRefBoneInfo[trackIndex];
+            BonesTransform[boneIndex] = new Transform[sequence.NumFrames];
+            for (int frame = 0; frame < BonesTransform[boneIndex].Length; frame++)
+            {
+                // calculate position for not animated bones based on the parent???
+                BonesTransform[boneIndex][frame] = skeleton.BonesTransformByIndex[boneIndex];
+            }
+        }
+
+        for (int trackIndex = 0; trackIndex < anim.TrackBonesInfo.Length; trackIndex++)
+        {
+            var bone = anim.TrackBonesInfo[trackIndex];
             if (!skeleton.BonesIndicesByLoweredName.TryGetValue(bone.Name.Text.ToLower(), out var boneIndices))
                 continue;
 
@@ -52,23 +63,57 @@ public class Sequence : IDisposable
                 if (frame < sequence.Tracks[trackIndex].KeyScale.Length)
                     boneScale = sequence.Tracks[trackIndex].KeyScale[frame];
 
+                switch (anim.BoneModes[trackIndex])
+                {
+                    case EBoneTranslationRetargetingMode.Skeleton:
+                    {
+                        var targetTransform = sequence.RetargetBasePose?[trackIndex] ?? anim.BonePositions[trackIndex];
+                        bonePosition = targetTransform.Translation;
+                        break;
+                    }
+                    case EBoneTranslationRetargetingMode.AnimationScaled:
+                    {
+                        var sourceTranslationLength = (originalTransform.Position / Constants.SCALE_DOWN_RATIO).Size();
+                        if (sourceTranslationLength > UnrealMath.KindaSmallNumber)
+                        {
+                            var targetTranslationLength = sequence.RetargetBasePose?[trackIndex].Translation.Size() ?? anim.BonePositions[trackIndex].Translation.Size();
+                            bonePosition.Scale(targetTranslationLength / sourceTranslationLength);
+                        }
+                        break;
+                    }
+                    case EBoneTranslationRetargetingMode.AnimationRelative:
+                    {
+                        // https://github.com/EpicGames/UnrealEngine/blob/cdaec5b33ea5d332e51eee4e4866495c90442122/Engine/Source/Runtime/Engine/Private/Animation/AnimationRuntime.cpp#L2586
+                        var refPoseTransform  = sequence.RetargetBasePose?[trackIndex] ?? anim.BonePositions[trackIndex];
+                        break;
+                    }
+                    case EBoneTranslationRetargetingMode.OrientAndScale:
+                    {
+                        var sourceSkelTrans = originalTransform.Position / Constants.SCALE_DOWN_RATIO;
+                        var targetSkelTrans = sequence.RetargetBasePose?[trackIndex].Translation ?? anim.BonePositions[trackIndex].Translation;
+
+                        if (!sourceSkelTrans.Equals(targetSkelTrans))
+                        {
+                            var sourceSkelTransLength = sourceSkelTrans.Size();
+                            var targetSkelTransLength = targetSkelTrans.Size();
+                            if (!UnrealMath.IsNearlyZero(sourceSkelTransLength * targetSkelTransLength))
+                            {
+                                var sourceSkelTransDir = sourceSkelTrans / sourceSkelTransLength;
+                                var targetSkelTransDir = targetSkelTrans / targetSkelTransLength;
+
+                                var deltaRotation = FQuat.FindBetweenNormals(sourceSkelTransDir, targetSkelTransDir);
+                                var scale = targetSkelTransLength / sourceSkelTransLength;
+                                bonePosition = deltaRotation.RotateVector(bonePosition) * scale;
+                            }
+                        }
+                        break;
+                    }
+                }
+
                 // revert FixRotationKeys
                 if (trackIndex > 0) boneOrientation.Conjugate();
-
                 bonePosition *= Constants.SCALE_DOWN_RATIO;
 
-                // switch (boneModes[trackIndex])
-                // {
-                //     case EBoneRetargetingMode.Animation:
-                //     case EBoneRetargetingMode.Mesh:
-                //     case EBoneRetargetingMode.AnimationScaled:
-                //     case EBoneRetargetingMode.AnimationRelative:
-                //     case EBoneRetargetingMode.OrientAndScale:
-                //     case EBoneRetargetingMode.Count:
-                //     default:
-                //         break;
-                // }
-
                 BonesTransform[boneIndices.Index][frame] = new Transform
                 {
                     Relation = boneIndices.ParentIndex >= 0 ? BonesTransform[boneIndices.ParentIndex][frame].Matrix : originalTransform.Relation,
@@ -80,6 +125,10 @@ public class Sequence : IDisposable
         }
     }
 
+    public void Dispose()
+    {
+
+    }
 
     private readonly float _height = 20.0f;
     public void DrawSequence(ImDrawListPtr drawList, float x, float y, Vector2 ratio, int index, uint col)
@@ -92,9 +141,4 @@ public class Sequence : IDisposable
         drawList.AddText(p1 with { X = p1.X + 2.5f }, 0xFF000000, Name);
         drawList.PopClipRect();
     }
-
-    public void Dispose()
-    {
-        throw new NotImplementedException();
-    }
 }

FModel/Views/Snooper/Models/Animations/Skeleton.cs

@@ -3,10 +3,8 @@ using System.Collections.Generic;
 using System.Numerics;
 using CUE4Parse_Conversion.Animations;
 using CUE4Parse.UE4.Assets.Exports.Animation;
-using CUE4Parse.UE4.Objects.UObject;
 using FModel.Views.Snooper.Buffers;
 using OpenTK.Graphics.OpenGL4;
-using Serilog;
 
 namespace FModel.Views.Snooper.Models.Animations;
 
@@ -21,12 +19,10 @@ public class Skeleton : IDisposable
     private int _handle;
     private BufferObject<Matrix4x4> _ssbo;
 
-    public readonly USkeleton UnrealSkeleton;
+    public string Name;
-    public readonly bool IsLoaded;
-
     public readonly Dictionary<string, BoneIndice> BonesIndicesByLoweredName;
     public readonly Dictionary<int, Transform> BonesTransformByIndex;
-    public readonly Matrix4x4[] InvertedBonesMatrixByIndex;
+    public readonly int BoneCount;
 
     public Animation Anim;
     public bool HasAnim => Anim != null;
@@ -35,31 +31,16 @@ public class Skeleton : IDisposable
     {
         BonesIndicesByLoweredName = new Dictionary<string, BoneIndice>();
         BonesTransformByIndex = new Dictionary<int, Transform>();
-        InvertedBonesMatrixByIndex = Array.Empty<Matrix4x4>();
     }
 
-    public Skeleton(FPackageIndex package, FReferenceSkeleton referenceSkeleton) : this()
+    public Skeleton(FReferenceSkeleton referenceSkeleton) : this()
     {
-        UnrealSkeleton = package.Load<USkeleton>();
-        IsLoaded = UnrealSkeleton != null;
-        if (!IsLoaded) return;
-
         for (int boneIndex = 0; boneIndex < referenceSkeleton.FinalRefBoneInfo.Length; boneIndex++)
         {
             var info = referenceSkeleton.FinalRefBoneInfo[boneIndex];
             BonesIndicesByLoweredName[info.Name.Text.ToLower()] = new BoneIndice { Index = boneIndex, ParentIndex = info.ParentIndex };
         }
 
-#if DEBUG
-        for (int trackIndex = 0; trackIndex < UnrealSkeleton.ReferenceSkeleton.FinalRefBoneInfo.Length; trackIndex++)
-        {
-            var bone = UnrealSkeleton.ReferenceSkeleton.FinalRefBoneInfo[trackIndex];
-            if (!BonesIndicesByLoweredName.TryGetValue(bone.Name.Text.ToLower(), out _))
-                Log.Warning($"Bone Mismatch: {bone.Name.Text} ({trackIndex}) is not present in the mesh's skeleton");
-        }
-#endif
-
-        InvertedBonesMatrixByIndex = new Matrix4x4[BonesIndicesByLoweredName.Count];
         foreach (var boneIndices in BonesIndicesByLoweredName.Values)
         {
             var bone = referenceSkeleton.FinalRefBonePose[boneIndices.Index];
@@ -77,11 +58,10 @@ public class Skeleton : IDisposable
                 parentTransform = new Transform { Relation = Matrix4x4.Identity };
 
             boneTransform.Relation = parentTransform.Matrix;
-            Matrix4x4.Invert(boneTransform.Matrix, out var inverted);
-
             BonesTransformByIndex[boneIndices.Index] = boneTransform;
-            InvertedBonesMatrixByIndex[boneIndices.Index] = inverted;
         }
+
+        BoneCount = BonesTransformByIndex.Count;
     }
 
     public void SetAnimation(CAnimSet anim, bool rotationOnly)
@@ -92,25 +72,23 @@ public class Skeleton : IDisposable
     public void Setup()
     {
         _handle = GL.CreateProgram();
-        _ssbo = new BufferObject<Matrix4x4>(InvertedBonesMatrixByIndex.Length, BufferTarget.ShaderStorageBuffer);
+
+        _ssbo = new BufferObject<Matrix4x4>(BoneCount, BufferTarget.ShaderStorageBuffer);
+        for (int boneIndex = 0; boneIndex < BoneCount; boneIndex++)
+            _ssbo.Update(boneIndex, Matrix4x4.Identity);
+        _ssbo.BindBufferBase(1);
     }
 
     public void UpdateMatrices(float deltaSeconds)
     {
-        if (!IsLoaded) return;
+        if (!HasAnim) return;
 
         _ssbo.BindBufferBase(1);
-        if (!HasAnim)
+
-        {
+        Anim.Update(deltaSeconds);
-            for (int boneIndex = 0; boneIndex < InvertedBonesMatrixByIndex.Length; boneIndex++)
+        for (int boneIndex = 0; boneIndex < BoneCount; boneIndex++)
-                _ssbo.Update(boneIndex, Matrix4x4.Identity);
+            _ssbo.Update(boneIndex, Anim.InterpolateBoneTransform(boneIndex));
-        }
+
-        else
-        {
-            Anim.Update(deltaSeconds);
-            for (int boneIndex = 0; boneIndex < InvertedBonesMatrixByIndex.Length; boneIndex++)
-                _ssbo.Update(boneIndex, InvertedBonesMatrixByIndex[boneIndex] * Anim.InterpolateBoneTransform(boneIndex));
-        }
         _ssbo.Unbind();
     }
 

FModel/Views/Snooper/Models/Model.cs

@@ -80,7 +80,7 @@ public class Model : IDisposable
     public Material[] Materials;
     public bool TwoSided;
 
-    public bool HasSkeleton => Skeleton is { IsLoaded: true };
+    public bool HasSkeleton => Skeleton != null;
     public readonly Skeleton Skeleton;
 
     public bool HasSockets => Sockets.Length > 0;
@@ -137,11 +137,15 @@ public class Model : IDisposable
     private Model(USkeletalMesh export, CSkeletalMesh skeletalMesh, Transform transform) : this(export, export.Materials, skeletalMesh.LODs, transform)
     {
         Box = skeletalMesh.BoundingBox * Constants.SCALE_DOWN_RATIO;
-        Skeleton = new Skeleton(export.Skeleton, export.ReferenceSkeleton);
+        Skeleton = new Skeleton(export.ReferenceSkeleton);
 
         var sockets = new List<FPackageIndex>();
         sockets.AddRange(export.Sockets);
-        if (HasSkeleton) sockets.AddRange(Skeleton.UnrealSkeleton.Sockets);
+        if (HasSkeleton && export.Skeleton.TryLoad(out USkeleton skeleton))
+        {
+            Skeleton.Name = skeleton.Name;
+            sockets.AddRange(skeleton.Sockets);
+        }
 
         Sockets = new Socket[sockets.Count];
         for (int i = 0; i < Sockets.Length; i++)

FModel/Views/Snooper/Renderer.cs

@@ -92,16 +92,16 @@ public class Renderer : IDisposable
 
     public void Animate(UObject anim)
     {
-        if (!Options.TryGetModel(out var model) || !model.Skeleton.IsLoaded)
+        if (!Options.TryGetModel(out var model) || !model.HasSkeleton)
             return;
 
         switch (anim)
         {
-            case UAnimSequence animSequence:
+            case UAnimSequence animSequence when animSequence.Skeleton.TryLoad(out USkeleton skeleton):
-                model.Skeleton.SetAnimation(model.Skeleton.UnrealSkeleton.ConvertAnims(animSequence), AnimateWithRotationOnly);
+                model.Skeleton.SetAnimation(skeleton.ConvertAnims(animSequence), AnimateWithRotationOnly);
                 break;
-            case UAnimMontage animMontage:
+            case UAnimMontage animMontage when animMontage.Skeleton.TryLoad(out USkeleton skeleton):
-                model.Skeleton.SetAnimation(model.Skeleton.UnrealSkeleton.ConvertAnims(animMontage), AnimateWithRotationOnly);
+                model.Skeleton.SetAnimation(skeleton.ConvertAnims(animMontage), AnimateWithRotationOnly);
                 break;
         }
         Options.AnimateMesh(false);

FModel/Views/Snooper/SnimGui.cs

@@ -453,8 +453,8 @@ Snooper aims to give an accurate preview of models, materials, skeletal animatio
                     Layout("Guid");ImGui.Text($"  :  {s.Renderer.Options.SelectedModel.ToString(EGuidFormats.UniqueObjectGuid)}");
                     if (model.HasSkeleton)
                     {
-                        Layout("Skeleton");ImGui.Text($"  :  {model.Skeleton.UnrealSkeleton.Name}");
+                        Layout("Skeleton");ImGui.Text($"  :  {model.Skeleton.Name}");
-                        Layout("Bones");ImGui.Text($"  :  x{model.Skeleton.InvertedBonesMatrixByIndex.Length}");
+                        Layout("Bones");ImGui.Text($"  :  x{model.Skeleton.BoneCount}");
                     }
                     else
                     {