From 0341725770cc1e073580a15281e4d607dc1fc445 Mon Sep 17 00:00:00 2001 From: Masataka SUMI Date: Mon, 1 Feb 2021 22:29:42 +0900 Subject: [PATCH] Implements expression overriding https://github.com/vrm-c/vrm-specification/pull/196 --- .../Components/VRM10ControllerEditor.cs | 24 ++++-- .../Expression/DefaultExpressionValidator.cs | 84 ++++++++++++++++--- .../Expression/LookAtEyeDirection.cs | 10 +++ .../Runtime/Components/VRM10Controller.cs | 5 +- .../Components/VRM10ControllerExpression.cs | 27 ++++-- .../Components/VRM10ControllerLookAt.cs | 7 +- 6 files changed, 129 insertions(+), 28 deletions(-) diff --git a/Assets/VRM10/Editor/Components/VRM10ControllerEditor.cs b/Assets/VRM10/Editor/Components/VRM10ControllerEditor.cs index e77875e49..8e4c425d3 100644 --- a/Assets/VRM10/Editor/Components/VRM10ControllerEditor.cs +++ b/Assets/VRM10/Editor/Components/VRM10ControllerEditor.cs @@ -136,6 +136,9 @@ namespace UniVRM10 } serializedObject.Update(); + + // Setup runtime function. + m_target.Setup(); // base.OnInspectorGUI(); switch (_tab) @@ -176,14 +179,6 @@ namespace UniVRM10 void ExpressionGUI() { EditorGUILayout.Space(); - EditorGUILayout.LabelField("Override rates", EditorStyles.boldLabel); - EditorGUI.BeginDisabledGroup(true); - { - EditorGUILayout.Slider("Blink override rate", m_target.Expression.BlinkOverrideRate, 0f, 1f); - EditorGUILayout.Slider("LookAt override rate", m_target.Expression.LookAtOverrideRate, 0f, 1f); - EditorGUILayout.Slider("Mouth override rate", m_target.Expression.MouthOverrideRate, 0f, 1f); - } - EditorGUI.EndDisabledGroup(); if (!Application.isPlaying) { @@ -197,6 +192,9 @@ namespace UniVRM10 if (m_sliders != null) { + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Expression Weights", EditorStyles.boldLabel); + var sliders = m_sliders.Select(x => x.Slider()); foreach (var slider in sliders) { @@ -204,6 +202,16 @@ namespace UniVRM10 } m_target.Expression.SetWeights(m_expressionKeyWeights); } + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Override rates", EditorStyles.boldLabel); + EditorGUI.BeginDisabledGroup(true); + { + EditorGUILayout.Slider("Blink override rate", m_target.Expression.BlinkOverrideRate, 0f, 1f); + EditorGUILayout.Slider("LookAt override rate", m_target.Expression.LookAtOverrideRate, 0f, 1f); + EditorGUILayout.Slider("Mouth override rate", m_target.Expression.MouthOverrideRate, 0f, 1f); + } + EditorGUI.EndDisabledGroup(); } void OnSceneGUI() diff --git a/Assets/VRM10/Runtime/Components/Expression/DefaultExpressionValidator.cs b/Assets/VRM10/Runtime/Components/Expression/DefaultExpressionValidator.cs index de6aa8de0..e3bdb3a8b 100644 --- a/Assets/VRM10/Runtime/Components/Expression/DefaultExpressionValidator.cs +++ b/Assets/VRM10/Runtime/Components/Expression/DefaultExpressionValidator.cs @@ -1,37 +1,101 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; using VrmLib; namespace UniVRM10 { public sealed class DefaultExpressionValidator : IExpressionValidator { + private readonly Dictionary _expressions; + private DefaultExpressionValidator(VRM10ExpressionAvatar expressionAvatar) { - + _expressions = expressionAvatar.Clips.ToDictionary(ExpressionKey.CreateFromClip, x => x); } public void Validate(IReadOnlyDictionary inputWeights, IDictionary actualWeights, LookAtEyeDirection inputEyeDirection, out LookAtEyeDirection actualEyeDirection, out float blinkOverrideRate, out float lookAtOverrideRate, out float mouthOverrideRate) { - // weights + // override rate + blinkOverrideRate = 0f; + lookAtOverrideRate = 0f; + mouthOverrideRate = 0f; + + // 1. Set weights and Accumulate override rates. foreach (var (key, weight) in inputWeights) { if (!actualWeights.ContainsKey(key)) { actualWeights.Add(key, weight); } - + // Set weight. actualWeights[key] = weight; + + // Get expression. + if (!_expressions.ContainsKey(key)) continue; + var expression = _expressions[key]; + + // Override rate without targeting myself. + if (!key.IsBlink) + { + blinkOverrideRate = Mathf.Max(blinkOverrideRate, GetOverrideRate(expression.OverrideBlink, weight)); + } + if (!key.IsLookAt) + { + lookAtOverrideRate = Mathf.Max(lookAtOverrideRate, GetOverrideRate(expression.OverrideLookAt, weight)); + } + if (!key.IsMouth) + { + mouthOverrideRate = Mathf.Max(mouthOverrideRate, GetOverrideRate(expression.OverrideMouth, weight)); + } } - // eye direction - actualEyeDirection = inputEyeDirection; + // 2. Saturate rate. + blinkOverrideRate = Mathf.Clamp01(blinkOverrideRate); + lookAtOverrideRate = Mathf.Clamp01(lookAtOverrideRate); + mouthOverrideRate = Mathf.Clamp01(mouthOverrideRate); - // override rate - blinkOverrideRate = 0f; - lookAtOverrideRate = 0f; - mouthOverrideRate = 0f; + var blinkMultiplier = 1f - blinkOverrideRate; + var lookAtMultiplier = 1f - lookAtOverrideRate; + var mouthMultiplier = 1f - mouthOverrideRate; + + // 3. Set procedural key's weights. + foreach (var (key, weight) in inputWeights) + { + if (key.IsBlink) + { + actualWeights[key] = weight * blinkMultiplier; + } + else if (key.IsLookAt) + { + actualWeights[key] = weight * lookAtMultiplier; + } + else if (key.IsMouth) + { + actualWeights[key] = weight * mouthMultiplier; + } + } + + // 4. eye direction + actualEyeDirection = LookAtEyeDirection.Multiply(inputEyeDirection, 1f - lookAtOverrideRate); + } + + private float GetOverrideRate(ExpressionOverrideType type, float weight) + { + switch (type) + { + case ExpressionOverrideType.None: + return 0f; + case ExpressionOverrideType.Block: + return 1f; + case ExpressionOverrideType.Blend: + return weight; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, null); + } } public sealed class Factory : IExpressionValidatorFactory diff --git a/Assets/VRM10/Runtime/Components/Expression/LookAtEyeDirection.cs b/Assets/VRM10/Runtime/Components/Expression/LookAtEyeDirection.cs index 02ffd9359..e3d486f41 100644 --- a/Assets/VRM10/Runtime/Components/Expression/LookAtEyeDirection.cs +++ b/Assets/VRM10/Runtime/Components/Expression/LookAtEyeDirection.cs @@ -29,5 +29,15 @@ RightYaw = rightYaw; RightPitch = rightPitch; } + + public static LookAtEyeDirection Multiply(LookAtEyeDirection a, float b) + { + return new LookAtEyeDirection( + a.LeftYaw * b, + a.LeftPitch * b, + a.RightYaw * b, + a.RightPitch * b + ); + } } } \ No newline at end of file diff --git a/Assets/VRM10/Runtime/Components/VRM10Controller.cs b/Assets/VRM10/Runtime/Components/VRM10Controller.cs index 5456ddf0b..1a5dd1781 100644 --- a/Assets/VRM10/Runtime/Components/VRM10Controller.cs +++ b/Assets/VRM10/Runtime/Components/VRM10Controller.cs @@ -52,7 +52,10 @@ namespace UniVRM10 void OnDestroy() { - Expression.Dispose(); + if (Expression != null) + { + Expression.Restore(); + } if (ModelAsset != null) { diff --git a/Assets/VRM10/Runtime/Components/VRM10ControllerExpression.cs b/Assets/VRM10/Runtime/Components/VRM10ControllerExpression.cs index bbe31c675..9c3dffe8e 100644 --- a/Assets/VRM10/Runtime/Components/VRM10ControllerExpression.cs +++ b/Assets/VRM10/Runtime/Components/VRM10ControllerExpression.cs @@ -7,7 +7,7 @@ using VrmLib; namespace UniVRM10 { [Serializable] - public sealed class VRM10ControllerExpression : IDisposable + public sealed class VRM10ControllerExpression { public static IExpressionValidatorFactory ExpressionValidatorFactory = new DefaultExpressionValidator.Factory(); @@ -31,12 +31,6 @@ namespace UniVRM10 public float LookAtOverrideRate { get; private set; } public float MouthOverrideRate { get; private set; } - public void Dispose() - { - _merger?.RestoreMaterialInitialValues(); - _eyeDirectionApplicable?.Restore(); - } - internal void Setup(Transform transform, ILookAtEyeDirectionProvider eyeDirectionProvider, ILookAtEyeDirectionApplicable eyeDirectionApplicable) { if (ExpressionAvatar == null) @@ -45,14 +39,31 @@ namespace UniVRM10 return; } + Restore(); + _merger = new ExpressionMerger(ExpressionAvatar.Clips, transform); _keys = ExpressionAvatar.Clips.Select(ExpressionKey.CreateFromClip).ToList(); + var oldInputWeights = _inputWeights; _inputWeights = _keys.ToDictionary(x => x, x => 0f); + foreach (var key in _keys) + { + // remain user input weights. + if (oldInputWeights.ContainsKey(key)) _inputWeights[key] = oldInputWeights[key]; + } _actualWeights = _keys.ToDictionary(x => x, x => 0f); _validator = ExpressionValidatorFactory.Create(ExpressionAvatar); _eyeDirectionProvider = eyeDirectionProvider; _eyeDirectionApplicable = eyeDirectionApplicable; } + + internal void Restore() + { + _merger?.RestoreMaterialInitialValues(); + _merger = null; + + _eyeDirectionApplicable?.Restore(); + _eyeDirectionApplicable = null; + } public void Process() { @@ -107,7 +118,7 @@ namespace UniVRM10 private void Apply() { // 1. Get eye direction from provider. - _inputEyeDirection = _eyeDirectionProvider.EyeDirection; + _inputEyeDirection = _eyeDirectionProvider?.EyeDirection ?? default; // 2. Validate user input, and Output as actual weights. _validator.Validate(_inputWeights, _actualWeights, diff --git a/Assets/VRM10/Runtime/Components/VRM10ControllerLookAt.cs b/Assets/VRM10/Runtime/Components/VRM10ControllerLookAt.cs index 16cfd9d95..dff543430 100644 --- a/Assets/VRM10/Runtime/Components/VRM10ControllerLookAt.cs +++ b/Assets/VRM10/Runtime/Components/VRM10ControllerLookAt.cs @@ -136,7 +136,12 @@ namespace UniVRM10 m_head = head; m_leftEye = animator.GetBoneTransform(HumanBodyBones.LeftEye); m_rightEye = animator.GetBoneTransform(HumanBodyBones.RightEye); - if (Gaze == null) + + var isRuntimeAsset = true; +#if UNITY_EDITOR + isRuntimeAsset = Application.isPlaying && !PrefabUtility.IsPartOfAnyPrefab(m_head); +#endif + if (isRuntimeAsset && LookAtTargetType == LookAtTargetTypes.CalcYawPitchToGaze && Gaze == null) { Gaze = new GameObject().transform; Gaze.name = "__LOOKAT_GAZE__";