From 3b1ed6fef904af8e7568d3e6f9b1489a68dfd700 Mon Sep 17 00:00:00 2001 From: ousttrue Date: Fri, 8 Feb 2019 22:19:29 +0900 Subject: [PATCH 1/4] optimize ListTreeNode Children for large JSON --- Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs index 0c1fef64a..7c30b6380 100644 --- a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs +++ b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs @@ -338,7 +338,7 @@ namespace UniJSON { get { - for (int i = 0; i < m_Values.Count; ++i) + for (int i = ValueIndex; i < m_Values.Count; ++i) { if (m_Values[i].ParentIndex == ValueIndex) { From 022308056abe2d4e308a5fd26959043b8d675bf5 Mon Sep 17 00:00:00 2001 From: ousttrue Date: Sun, 10 Feb 2019 00:56:20 +0900 Subject: [PATCH 2/4] Remove unused code --- Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs index 1baae04e3..2789e9c48 100644 --- a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs +++ b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs @@ -4,11 +4,6 @@ using System.Collections.Generic; namespace UniJSON { - public class JsonParseResult - { - public List Values = new List(); - } - public static class JsonParser { static ValueNodeType GetValueType(Utf8String segment) From 08ff5afae8fec6e2cc71effb7be3bba1746532f7 Mon Sep 17 00:00:00 2001 From: ousttrue Date: Sun, 10 Feb 2019 02:57:03 +0900 Subject: [PATCH 3/4] Refactoring JsonParser Use ListTreeNode.AddValue to building tree --- Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs | 95 ++++++++----------- .../Scripts/ListTreeNode/ListTreeNode.cs | 10 +- 2 files changed, 47 insertions(+), 58 deletions(-) diff --git a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs index 2789e9c48..f67b01119 100644 --- a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs +++ b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; namespace UniJSON @@ -16,7 +15,7 @@ namespace UniJSON case 't': return ValueNodeType.Boolean; case 'f': return ValueNodeType.Boolean; case 'n': - if (segment.ByteLength >= 2 && Char.ToLower((char) segment[1]) == 'a') + if (segment.ByteLength >= 2 && Char.ToLower((char)segment[1]) == 'a') { return ValueNodeType.NaN; } @@ -27,7 +26,7 @@ namespace UniJSON return ValueNodeType.Infinity; case '-': - if (segment.ByteLength >= 2 && Char.ToLower((char) segment[1]) == 'i') + if (segment.ByteLength >= 2 && Char.ToLower((char)segment[1]) == 'i') { return ValueNodeType.MinusInfinity; } @@ -65,7 +64,7 @@ namespace UniJSON /// /// /// - static JsonValue ParsePrimitive(Utf8String segment, ValueNodeType valueType, int parentIndex) + static ListTreeNode ParsePrimitive(ListTreeNode tree, Utf8String segment, ValueNodeType valueType) { int i = 1; for (; i < segment.ByteLength; ++i) @@ -80,15 +79,15 @@ namespace UniJSON break; } } - return new JsonValue(segment.Subbytes(0, i), valueType, parentIndex); + return tree.AddValue(segment.Subbytes(0, i).Bytes, valueType); } - static JsonValue ParseString(Utf8String segment, int parentIndex) + static ListTreeNode ParseString(ListTreeNode tree, Utf8String segment) { int pos; if (segment.TrySearchAscii((Byte)'"', 1, out pos)) { - return new JsonValue(segment.Subbytes(0, pos + 1), ValueNodeType.String, parentIndex); + return tree.AddValue(segment.Subbytes(0, pos + 1).Bytes, ValueNodeType.String); } else { @@ -96,8 +95,10 @@ namespace UniJSON } } - static Utf8String ParseArray(Utf8String segment, List values, int parentIndex) + static ListTreeNode ParseArray(ListTreeNode tree, Utf8String segment) { + var array = tree.AddValue(segment.Bytes, ValueNodeType.Array); + var closeChar = ']'; bool isFirst = true; var current = segment.Subbytes(1); @@ -147,15 +148,22 @@ namespace UniJSON } // value - var value = Parse(current, values, parentIndex); - current = current.Subbytes(value.Segment.ByteLength); + var child = Parse(array, current); + current = current.Subbytes(child.Value.Segment.ByteLength); } - return current; + // fix array range + var count = current.Bytes.Offset + 1 - segment.Bytes.Offset; + var arraySegment = segment.Subbytes(0, count); + array.SetValue(new JsonValue(arraySegment, ValueNodeType.Array, array.Value.ParentIndex)); + + return array; } - static Utf8String ParseObject(Utf8String segment, List values, int parentIndex) + static ListTreeNode ParseObject(ListTreeNode tree, Utf8String segment) { + var obj = tree.AddValue(segment.Bytes, ValueNodeType.Object); + var closeChar = '}'; bool isFirst = true; var current = segment.Subbytes(1); @@ -204,12 +212,12 @@ namespace UniJSON } // key - var key = Parse(current, values, parentIndex); - if (key.ValueType != ValueNodeType.String) + var key = Parse(obj, current); + if (!key.IsString()) { - throw new ParserException("object key must string: " + key.Segment); + throw new ParserException("object key must string: " + key.Value.Segment); } - current = current.Subbytes(key.Segment.ByteLength); + current = current.Subbytes(key.Value.Segment.ByteLength); // search ':' int valuePos; @@ -230,14 +238,19 @@ namespace UniJSON } // value - var value = Parse(current, values, parentIndex); - current = current.Subbytes(value.Segment.ByteLength); + var value = Parse(obj, current); + current = current.Subbytes(value.Value.Segment.ByteLength); } - return current; + // fix obj range + var count = current.Bytes.Offset + 1 - segment.Bytes.Offset; + var objSegment = segment.Subbytes(0, count); + obj.SetValue(new JsonValue(objSegment, ValueNodeType.Object, obj.Value.ParentIndex)); + + return obj; } - static JsonValue Parse(Utf8String segment, List values, int parentIndex) + public static ListTreeNode Parse(ListTreeNode tree, Utf8String segment) { // skip white space int pos; @@ -257,38 +270,16 @@ namespace UniJSON case ValueNodeType.NaN: case ValueNodeType.Infinity: case ValueNodeType.MinusInfinity: - { - var value= ParsePrimitive(segment, valueType, parentIndex); - values.Add(value); - return value; - } + return ParsePrimitive(tree, segment, valueType); case ValueNodeType.String: - { - var value= ParseString(segment, parentIndex); - values.Add(value); - return value; - } + return ParseString(tree, segment); case ValueNodeType.Array: // fall through - { - var index = values.Count; - values.Add(new JsonValue()); // placeholder - var current = ParseArray(segment, values, index); - values[index] = new JsonValue(segment.Subbytes(0, current.Bytes.Offset + 1 - segment.Bytes.Offset), - ValueNodeType.Array, parentIndex); - return values[index]; - } + return ParseArray(tree, segment); case ValueNodeType.Object: // fall through - { - var index = values.Count; - values.Add(new JsonValue()); // placeholder - var current=ParseObject(segment, values, index); - values[index] = new JsonValue(segment.Subbytes(0, current.Bytes.Offset + 1 - segment.Bytes.Offset), - ValueNodeType.Object, parentIndex); - return values[index]; - } + return ParseObject(tree, segment); default: throw new NotImplementedException(); @@ -302,17 +293,7 @@ namespace UniJSON public static ListTreeNode Parse(Utf8String json) { - var result = new List(); - var value = Parse(json, result, -1); - if (value.ValueType != ValueNodeType.Array && value.ValueType != ValueNodeType.Object) - { - result.Add(value); - return new ListTreeNode(result); - } - else - { - return new ListTreeNode(result); - } + return Parse(default(ListTreeNode), json); } } } diff --git a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs index 7c30b6380..dc3f31407 100644 --- a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs +++ b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs @@ -408,9 +408,17 @@ namespace UniJSON m_Values.Add(default(T).Key(key, ValueIndex)); } - public void AddValue(ArraySegment bytes, ValueNodeType valueType) + public ListTreeNode AddValue(ArraySegment bytes, ValueNodeType valueType) { + if (m_Values == null) + { + // initialize empty tree + m_Values = new List(); + ValueIndex = -1; + } + var index = m_Values.Count; m_Values.Add(default(T).New(bytes, valueType, ValueIndex)); + return new ListTreeNode(m_Values, index); } #endregion } From 1ca7c20f49d1a1ef5217920fbd86676a8874be6a Mon Sep 17 00:00:00 2001 From: ousttrue Date: Sun, 10 Feb 2019 04:17:46 +0900 Subject: [PATCH 4/4] Add IListTreeItem.ChildCount for optimization of ListTreeNode.Children --- Assets/VRM/UniJSON/Scripts/ITreeNode.cs | 2 + Assets/VRM/UniJSON/Scripts/IValue.cs | 1 + Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs | 6 +-- Assets/VRM/UniJSON/Scripts/Json/JsonValue.cs | 14 +++++ .../Scripts/ListTreeNode/ListTreeNode.cs | 51 +++++++++++++++---- .../UniJSON/Scripts/MsgPack/MsgPackParser.cs | 43 +++++++--------- .../UniJSON/Scripts/MsgPack/MsgPackValue.cs | 16 +++++- Assets/VRM/UniJSON/Scripts/Toml/TomlValue.cs | 17 +++++++ 8 files changed, 112 insertions(+), 38 deletions(-) diff --git a/Assets/VRM/UniJSON/Scripts/ITreeNode.cs b/Assets/VRM/UniJSON/Scripts/ITreeNode.cs index b1682dd04..1f8edfbbf 100644 --- a/Assets/VRM/UniJSON/Scripts/ITreeNode.cs +++ b/Assets/VRM/UniJSON/Scripts/ITreeNode.cs @@ -20,5 +20,7 @@ namespace UniJSON public interface IListTreeItem { int ParentIndex { get; } + int ChildCount { get; } + void SetChildCount(int count); } } diff --git a/Assets/VRM/UniJSON/Scripts/IValue.cs b/Assets/VRM/UniJSON/Scripts/IValue.cs index 95b68f78e..b88341f8a 100644 --- a/Assets/VRM/UniJSON/Scripts/IValue.cs +++ b/Assets/VRM/UniJSON/Scripts/IValue.cs @@ -24,6 +24,7 @@ namespace UniJSON T Key(Utf8String key, int parentIndex); ValueNodeType ValueType { get; } ArraySegment Bytes { get; } + void SetBytesCount(int count); Boolean GetBoolean(); String GetString(); Utf8String GetUtf8String(); diff --git a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs index f67b01119..8936d8275 100644 --- a/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs +++ b/Assets/VRM/UniJSON/Scripts/Json/JsonParser.cs @@ -154,8 +154,7 @@ namespace UniJSON // fix array range var count = current.Bytes.Offset + 1 - segment.Bytes.Offset; - var arraySegment = segment.Subbytes(0, count); - array.SetValue(new JsonValue(arraySegment, ValueNodeType.Array, array.Value.ParentIndex)); + array.SetValueBytesCount(count); return array; } @@ -244,8 +243,7 @@ namespace UniJSON // fix obj range var count = current.Bytes.Offset + 1 - segment.Bytes.Offset; - var objSegment = segment.Subbytes(0, count); - obj.SetValue(new JsonValue(objSegment, ValueNodeType.Object, obj.Value.ParentIndex)); + obj.SetValueBytesCount(count); return obj; } diff --git a/Assets/VRM/UniJSON/Scripts/Json/JsonValue.cs b/Assets/VRM/UniJSON/Scripts/Json/JsonValue.cs index 0fb2e774e..97ddcef24 100644 --- a/Assets/VRM/UniJSON/Scripts/Json/JsonValue.cs +++ b/Assets/VRM/UniJSON/Scripts/Json/JsonValue.cs @@ -7,6 +7,10 @@ namespace UniJSON { public Utf8String Segment; public ArraySegment Bytes { get { return Segment.Bytes; } } + public void SetBytesCount(int count) + { + Segment = new Utf8String(new ArraySegment(Bytes.Array, Bytes.Offset, count)); + } public ValueNodeType ValueType { @@ -20,6 +24,16 @@ namespace UniJSON private set; } + int _childCount; + public int ChildCount + { + get { return _childCount; } + } + public void SetChildCount(int count) + { + _childCount = count; + } + public JsonValue(Utf8String segment, ValueNodeType valueType, int parentIndex) : this() { Segment = segment; diff --git a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs index dc3f31407..c3cc147ab 100644 --- a/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs +++ b/Assets/VRM/UniJSON/Scripts/ListTreeNode/ListTreeNode.cs @@ -303,10 +303,14 @@ namespace UniJSON /// /// This node index /// + int _valueIndex; public int ValueIndex { - get; - private set; + get + { + if (m_Values == null) return -1; + return _valueIndex; + } } public ListTreeNode Prev @@ -334,14 +338,21 @@ namespace UniJSON } #region Children + public int ChildCount + { + get { return Value.ChildCount; } + } + public IEnumerable> Children { get { - for (int i = ValueIndex; i < m_Values.Count; ++i) + int count = 0; + for (int i = ValueIndex; count < ChildCount && i < m_Values.Count; ++i) { if (m_Values[i].ParentIndex == ValueIndex) { + ++count; yield return new ListTreeNode(m_Values, i); } } @@ -372,7 +383,6 @@ namespace UniJSON } } #endregion - public bool HasParent { get @@ -399,27 +409,50 @@ namespace UniJSON public ListTreeNode(List values, int index = 0) : this() { m_Values = values; - ValueIndex = index; + _valueIndex = index; } #region JsonPointer - public void AddKey(Utf8String key) + public ListTreeNode AddKey(Utf8String key) { - m_Values.Add(default(T).Key(key, ValueIndex)); + return AddValue(default(T).Key(key, ValueIndex)); } public ListTreeNode AddValue(ArraySegment bytes, ValueNodeType valueType) + { + return AddValue(default(T).New(bytes, valueType, ValueIndex)); + } + + public ListTreeNode AddValue(T value) { if (m_Values == null) { // initialize empty tree m_Values = new List(); - ValueIndex = -1; + _valueIndex = -1; + } + else + { + IncrementChildCount(); } var index = m_Values.Count; - m_Values.Add(default(T).New(bytes, valueType, ValueIndex)); + m_Values.Add(value); return new ListTreeNode(m_Values, index); } + + void IncrementChildCount() + { + var value = Value; + value.SetChildCount(value.ChildCount + 1); + SetValue(value); + } + + public void SetValueBytesCount(int count) + { + var value = Value; + value.SetBytesCount(count); + SetValue(value); + } #endregion } } diff --git a/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackParser.cs b/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackParser.cs index 9d153f589..cf0e07f60 100644 --- a/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackParser.cs +++ b/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackParser.cs @@ -357,65 +357,60 @@ namespace UniJSON } } - static ArraySegment _Parse(ArraySegment bytes, List values, int parentIndex) + static ListTreeNode _Parse(ListTreeNode tree, ArraySegment bytes) { MsgPackType formatType = GetFormat(bytes); if (formatType.IsArray()) { - var index = values.Count; - var offset = bytes.Offset; - values.Add(new MsgPackValue(bytes, parentIndex)); + var array = tree.AddValue(bytes, ValueNodeType.Array); uint count; bytes = GetItemCount(bytes, formatType, out count); for (var i = 0; i < count; ++i) { - bytes = _Parse(bytes, values, index); + var child = _Parse(array, bytes); + bytes = bytes.Advance(child.Value.Bytes.Count); } - values[index] = new MsgPackValue( - new ArraySegment(bytes.Array, - offset, bytes.Offset - offset), - parentIndex); + array.SetValueBytesCount(bytes.Offset - array.Value.Bytes.Offset); + + return array; } else if (formatType.IsMap()) { - var index = values.Count; - var offset = bytes.Offset; - values.Add(new MsgPackValue(bytes, parentIndex)); + var obj = tree.AddValue(bytes, ValueNodeType.Object); uint count; bytes = GetItemCount(bytes, formatType, out count); for (var i = 0; i < count; ++i) { // key - bytes = _Parse(bytes, values, index); + var key = _Parse(obj, bytes); + bytes = bytes.Advance(key.Value.Bytes.Count); // value - bytes = _Parse(bytes, values, index); + var value = _Parse(obj, bytes); + bytes = bytes.Advance(value.Value.Bytes.Count); } - values[index] = new MsgPackValue( - new ArraySegment(bytes.Array, - offset, bytes.Offset - offset), - parentIndex); + obj.SetValueBytesCount(bytes.Offset - obj.Value.Bytes.Offset); + + return obj; } else { var body = GetBody(bytes, formatType); var headerSize = body.Offset - bytes.Offset; var size = headerSize + body.Count; - values.Add(new MsgPackValue(bytes.Take(size), parentIndex)); - bytes = bytes.Advance(size); + + var value = tree.AddValue(bytes.Take(size), ValueNodeType.Null); + return value; } - return bytes; } public static ListTreeNode Parse(ArraySegment bytes) { - var values = new List(); - _Parse(bytes, values, -1); - return new ListTreeNode(values); + return _Parse(default(ListTreeNode), bytes); } } } diff --git a/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackValue.cs b/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackValue.cs index b33c2bd5f..77c43f296 100644 --- a/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackValue.cs +++ b/Assets/VRM/UniJSON/Scripts/MsgPack/MsgPackValue.cs @@ -16,6 +16,10 @@ namespace UniJSON get; private set; } + public void SetBytesCount(int count) + { + Bytes = new ArraySegment(Bytes.Array, Bytes.Offset, count); + } public MsgPackType Format { @@ -71,6 +75,16 @@ namespace UniJSON } } + int _childCount; + public int ChildCount + { + get { return _childCount; } + } + public void SetChildCount(int count) + { + _childCount = count; + } + public MsgPackValue(ArraySegment segment, int parentIndex) : this() { Bytes = segment; @@ -79,7 +93,7 @@ namespace UniJSON public MsgPackValue New(ArraySegment bytes, ValueNodeType valueType, int parentIndex) { - throw new NotImplementedException(); + return new MsgPackValue(bytes, parentIndex); } public MsgPackValue Key(Utf8String key, int parentIndex) diff --git a/Assets/VRM/UniJSON/Scripts/Toml/TomlValue.cs b/Assets/VRM/UniJSON/Scripts/Toml/TomlValue.cs index 37ca04295..9a8391b66 100644 --- a/Assets/VRM/UniJSON/Scripts/Toml/TomlValue.cs +++ b/Assets/VRM/UniJSON/Scripts/Toml/TomlValue.cs @@ -62,7 +62,24 @@ namespace UniJSON } Utf8String m_segment; + public ArraySegment Bytes { get { return m_segment.Bytes; } } + public void SetBytesCount(int count) + { + throw new NotImplementedException(); + } + + public int ChildCount + { + get + { + throw new NotImplementedException(); + } + } + public void SetChildCount(int count) + { + throw new NotImplementedException(); + } public TomlValue(Utf8String segment, TomlValueType valueType, int parentIndex) : this() {