Leahnaya/kbinxml
1
0
Fork 0
mirror of https://github.com/mon/kbinxml.git synced 2026-08-12 20:05:31 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Switch to lxml, completely fix Unicode, add test cases

Browse Source
This commit is contained in:
William Toohey
2017-07-05 17:51:47 +10:00
parent be1c9f9a8c
commit d4514ea4be
6 changed files with 143 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -3,18 +3,18 @@
An encoder/decoder for Konami's binary XML format, used in some of their games.
### Setup:
`pip install bitarray`
`pip install bitarray lxml`
Python 2 only:
Additionally for Python 2:
`pip install future`
```python
In [1]: from kbinxml import KBinXML
In [2]: text = KBinXML('<?xml version="1.0"?><root __type="str">Hello, world!</root>')
In [3]: text.to_binary()
Out[4]: '\xa0B\x80\x7f\x00\x00\x00\x08\x0b\x04\xdfM9\xfe\xff\x00\x00\x00\x00\x14\x00\x00\x00\x0eHello, world!\x00\x00\x00'
Out[4]: b'\xa0B\x80\x7f\x00\x00\x00\x08\x0b\x04\xdfM9\xfe\xff\x00\x00\x00\x00\x14\x00\x00\x00\x0eHello, world!\x00\x00\x00'
In [5]: bin = KBinXML(Out[4])
In [6]: bin.to_text()
Out[7]: '<?xml version="1.0" encoding="UTF_8"?>\n<root __type="str">Hello, world!</root>\n'
Out[7]: u'<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<root __type="str">Hello, world!</root>\n'
```

62
kbinxml.py
View File

@@ -1,11 +1,12 @@
# python 3 style, ints instead of b''
from builtins import bytes
from xml.dom import minidom
from struct import calcsize
import string
import sys
import operator
from io import BytesIO
import lxml.etree as etree
from bytebuffer import ByteBuffer
from sixbit import pack_sixbit, unpack_sixbit
from format_ids import xml_formats, xml_types
@@ -20,7 +21,7 @@ SIGNATURE = 0xA0
SIG_COMPRESSED = 0x42
SIG_UNCOMPRESSED = 0x45
XML_ENCODING = 'UTF_8'
XML_ENCODING = 'UTF-8'
BIN_ENCODING = 'SHIFT_JISX0213'
# NOTE: all of these are their python codec names
@@ -41,18 +42,22 @@ def debug_print(string):
class KBinXML():
def __init__(self, input):
if isinstance(input, minidom.Document):
if isinstance(input, etree._Element):
self.xml_doc = input
elif isinstance(input, etree._ElementTree):
self.xml_doc = input.getroot()
elif KBinXML.is_binary_xml(input):
self.from_binary(input)
else:
self.from_text(input)
def to_text(self):
return self.xml_doc.toprettyxml(indent = " ", encoding = XML_ENCODING)
# we decode again because I want unicode, dammit
return etree.tostring(self.xml_doc, pretty_print=True,
encoding=XML_ENCODING, xml_declaration=True).decode(XML_ENCODING)
def from_text(self, input):
self.xml_doc = minidom.parseString(input)
self.xml_doc = etree.parse(BytesIO(input)).getroot()
@staticmethod
def is_binary_xml(input):
@@ -121,28 +126,30 @@ class KBinXML():
self.dataBuf.realign_writes()
def _node_to_binary(self, node):
if node.nodeType == node.TEXT_NODE or node.nodeType == node.COMMENT_NODE:
return
nodeType = node.getAttribute('__type')
nodeType = node.attrib.get('__type')
if not nodeType:
nodeType = 'void'
# typeless tags with text become string
if node.text is not None and len(node.text.strip()) > 0:
nodeType = 'str'
else:
nodeType = 'void'
nodeId = xml_types[nodeType]
isArray = 0
count = node.getAttribute('__count')
count = node.attrib.get('__count')
if count:
count = int(count)
isArray = 64 # bit position for array flag
self.nodeBuf.append_u8(nodeId | isArray)
name = node.nodeName
name = node.tag
pack_sixbit(name, self.nodeBuf)
if nodeType != 'void':
fmt = xml_formats[nodeId]
val = node.firstChild.nodeValue
val = node.text
if fmt['name'] == 'bin':
data = bytes(bytearray.fromhex(val))
elif fmt['name'] == 'str':
@@ -161,14 +168,14 @@ class KBinXML():
self.data_append_aligned(data, fmt['type'], fmt['count'])
# for test consistency and to be more faithful, sort the attrs
sorted_attrs = sorted(node.attributes.items(), key=operator.itemgetter(0))
sorted_attrs = sorted(node.attrib.items(), key=operator.itemgetter(0))
for key, value in sorted_attrs:
if key not in ['__type', '__size', '__count']:
self.data_append_string(value)
self.nodeBuf.append_u8(xml_types['attr'])
pack_sixbit(key, self.nodeBuf)
for child in node.childNodes:
for child in node.iterchildren(tag=etree.Element):
self._node_to_binary(child)
# always has the isArray bit set
@@ -188,8 +195,7 @@ class KBinXML():
self.dataByteBuf = ByteBuffer(self.dataBuf.data)
self.dataWordBuf = ByteBuffer(self.dataBuf.data)
for child in self.xml_doc.childNodes:
self._node_to_binary(child)
self._node_to_binary(self.xml_doc)
# always has the isArray bit set
self.nodeBuf.append_u8(xml_types['endSection'] | 64)
@@ -199,7 +205,7 @@ class KBinXML():
return bytes(header.data + self.nodeBuf.data + self.dataBuf.data)
def from_binary(self, input):
self.xml_doc = minidom.Document()
self.xml_doc = etree.Element('root')
node = self.xml_doc
self.nodeBuf = ByteBuffer(input)
@@ -249,10 +255,10 @@ class KBinXML():
if nodeType == xml_types['attr']:
value = self.data_grab_string()
node.setAttribute(name, value)
node.attrib[name] = value
elif nodeType == xml_types['nodeEnd']:
if node.parentNode:
node = node.parentNode
if node.getparent() is not None:
node = node.getparent()
elif nodeType == xml_types['endSection']:
nodesLeft = False
elif nodeType not in xml_formats:
@@ -263,14 +269,13 @@ class KBinXML():
if skip:
continue
child = self.xml_doc.createElement(name)
node.appendChild(child)
child = etree.SubElement(node, name)
node = child
if nodeType == xml_types['nodeStart']:
continue
node.setAttribute('__type', nodeFormat['name'])
node.attrib['__type'] = nodeFormat['name']
varCount = nodeFormat['count']
arrayCount = 1
@@ -279,7 +284,7 @@ class KBinXML():
isArray = True
elif isArray:
arrayCount = self.dataBuf.get_u32() // (calcsize(nodeFormat['type'] * varCount))
node.setAttribute('__count', str(arrayCount))
node.attrib['__count'] = str(arrayCount)
totalCount = arrayCount * varCount
if isArray:
@@ -289,14 +294,17 @@ class KBinXML():
data = self.data_grab_aligned(nodeFormat['type'], totalCount)
if nodeType == xml_types['binary']:
node.setAttribute('__size', str(totalCount))
node.attrib['__size'] = str(totalCount)
string = ''.join(('{0:02x}'.format(x) for x in data))
elif nodeType == xml_types['string']:
string = bytes(data[:-1]).decode(self.encoding)
else:
string = ' '.join(map(nodeFormat.get('toStr', str), data))
node.appendChild(self.xml_doc.createTextNode(string))
node.text = string
# because we need the 'real' root
self.xml_doc = self.xml_doc[0]
if __name__ == '__main__':
if len(sys.argv) != 2:
@@ -308,6 +316,6 @@ if __name__ == '__main__':
xml = KBinXML(input)
if KBinXML.is_binary_xml(input):
stdout.write(xml.to_text())
stdout.write(xml.to_text().encode('utf-8'))
else:
stdout.write(xml.to_binary())

28
test.py Normal file
View File

@@ -0,0 +1,28 @@
from kbinxml import KBinXML
# python 2/3 cross compat
from io import open
with open('testcases.xml', 'rb') as f:
xml_in = f.read()
with open('testcases_out.xml', 'r', encoding='UTF-8') as f:
expected_xml = f.read()
with open('testcases_out.kbin', 'rb') as f:
expected_bin = f.read()
k = KBinXML(xml_in)
kbin = k.to_binary()
if kbin != expected_bin:
with open('failed_test.bin', 'wb') as f:
f.write(kbin)
raise AssertionError('Binary output does not match, check failed_test.bin')
else:
print('XML -> Binary correct!')
backwards = KBinXML(kbin)
btext = backwards.to_text()
if btext != expected_xml:
with open('failed_test.xml', 'w', encoding='UTF-8') as f:
f.write(btext)
raise AssertionError('XML putput does not match, check failed_test.xml')
else:
print('Binary -> XML correct!')

2
testcases.xml
View File

@@ -5,6 +5,8 @@
<entry __type="ip4" __count="2">127.0.0.1 192.168.0.1</entry>
<!-- Attributes plus text -->
<entry attr="test" attr2="best" __type="str">Hello, world!</entry>
<!-- Testing encoding, plus __type-less should become string -->
<superstar><EFBFBD>V<EFBFBD>@<40>C<EFBFBD>X<EFBFBD>@<40>}<7D>C<EFBFBD>@<40><><EFBFBD>C<EFBFBD>t</superstar>
<!-- Testing 6bit conversion -->
<xXx_T4GG3R_xXx __type="3u8">8 9 10</xXx_T4GG3R_xXx>
<!-- Shouldn't have alignment issues from the 3u8 -->

BIN
testcases_out.kbin
View File

Binary file not shown.

147
testcases_out.xml
View File

@@ -1,75 +1,76 @@
<?xml version="1.0" encoding="UTF_8"?>
<?xml version='1.0' encoding='UTF-8'?>
<test>
<entry __count="2" __type="ip4">127.0.0.1 192.168.0.1</entry>
<entry __type="str" attr="test" attr2="best">Hello, world!</entry>
<xXx_T4GG3R_xXx __type="3u8">8 9 10</xXx_T4GG3R_xXx>
<aligned __type="u8">12</aligned>
<entry __size="4" __type="bin">deadbeef</entry>
<entry __size="4" __type="bin">deadbe7a</entry>
<entry __count="2" __type="3u8">1 2 3 1 2 3</entry>
<no_friends/>
<food_for_thought bread="tasty"/>
<entry __type="s8">123</entry>
<entry __type="u8">123</entry>
<entry __type="s16">123</entry>
<entry __type="u16">123</entry>
<entry __type="s32">123</entry>
<entry __type="u32">123</entry>
<entry __type="s64">123</entry>
<entry __type="u64">123</entry>
<entry __size="4" __type="bin">f00d1337</entry>
<entry __size="4" __type="bin">deadbeef</entry>
<entry __type="str">TEST STRING 2</entry>
<entry __type="str">TEST STRING</entry>
<entry __type="ip4">192.168.0.1</entry>
<entry __type="float">123.099998</entry>
<entry __type="float">123.099998</entry>
<entry __type="double">123.100000</entry>
<entry __type="double">123.100000</entry>
<entry __type="2s8">127 -23</entry>
<entry __count="2" __type="2u8">232 12 13 14</entry>
<entry __type="2s16">1232 -5433</entry>
<entry __type="2u16">345 3455</entry>
<entry __type="2s32">123123 -12672</entry>
<entry __type="2u32">5345 76575</entry>
<entry __type="2s64">5687 -5687</entry>
<entry __type="2s64">-6582345 52345</entry>
<entry __type="2u64">82345 52345</entry>
<entry __type="2u64">6582345 52345</entry>
<entry __type="2f">-32.000000 12.000000</entry>
<entry __type="2d">-32.000000 12.000000</entry>
<entry __type="2d">-32.000000 12.000000</entry>
<entry __type="3s8">28 127 -124</entry>
<entry __type="3u8">2 4 255</entry>
<entry __type="3s16">2842 12427 -14224</entry>
<entry __type="3u16">242 424 24255</entry>
<entry __type="3s32">2248 175627 -12244</entry>
<entry __type="3u32">24422 44242 25245</entry>
<entry __type="3s64">2898 10927 -12094</entry>
<entry __type="3u64">28908 8904 2955</entry>
<entry __type="3f">-32.000000 12.000000 3.000000</entry>
<entry __type="3d">-32.000000 12.000000 3.000000</entry>
<entry __type="4s8">24 28 127 -124</entry>
<entry __type="4u8">1 2 255 255</entry>
<entry __type="4s16">-34 -53 -23 34</entry>
<entry __type="4u16">123 123 154 654</entry>
<entry __type="4s32">12 132 656 -4565</entry>
<entry __type="4s32">12 132 -656 4565</entry>
<entry __type="4u32">12 132 656 4565</entry>
<entry __type="4u32">12 132 656 4565</entry>
<entry __type="4s64">-31231 5128741 -123222 129381231</entry>
<entry __type="4u64">31231 5128741 1280371203 129381231</entry>
<entry __type="4f">-32.000000 12.000000 3.000000 2.000000</entry>
<entry __type="4f">5.000000 52.000000 -12.000000 32.000000</entry>
<entry __type="4d">1.000000 3.000000 4.000000 -1.000000</entry>
<entry __type="vs8">1 3 5 7 12 16 46 76 93 123 0 -124 -123 -12 -42 -56</entry>
<entry __type="vu8">1 3 5 7 12 16 46 76 93 123 178 240 255 127 126 255</entry>
<entry __type="vs16">93 123 0 -124 -123 -12 -42 -56</entry>
<entry __type="vu16">16 46 76 93 123 178 240 255</entry>
<entry __type="bool">1</entry>
<entry __type="bool">1</entry>
<entry __type="2b">0 1</entry>
<entry __type="3b">1 0 1</entry>
<entry __type="4b">1 1 1 0</entry>
<entry __type="vb">1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0</entry>
<entry __type="ip4" __count="2">127.0.0.1 192.168.0.1</entry>
<entry __type="str" attr="test" attr2="best">Hello, world!</entry>
<superstar __type="str">シ　イス　マイ　ワイフ</superstar>
<xXx_T4GG3R_xXx __type="3u8">8 9 10</xXx_T4GG3R_xXx>
<aligned __type="u8">12</aligned>
<entry __type="bin" __size="4">deadbeef</entry>
<entry __type="bin" __size="4">deadbe7a</entry>
<entry __type="3u8" __count="2">1 2 3 1 2 3</entry>
<no_friends/>
<food_for_thought bread="tasty"/>
<entry __type="s8">123</entry>
<entry __type="u8">123</entry>
<entry __type="s16">123</entry>
<entry __type="u16">123</entry>
<entry __type="s32">123</entry>
<entry __type="u32">123</entry>
<entry __type="s64">123</entry>
<entry __type="u64">123</entry>
<entry __type="bin" __size="4">f00d1337</entry>
<entry __type="bin" __size="4">deadbeef</entry>
<entry __type="str">TEST STRING 2</entry>
<entry __type="str">TEST STRING</entry>
<entry __type="ip4">192.168.0.1</entry>
<entry __type="float">123.099998</entry>
<entry __type="float">123.099998</entry>
<entry __type="double">123.100000</entry>
<entry __type="double">123.100000</entry>
<entry __type="2s8">127 -23</entry>
<entry __type="2u8" __count="2">232 12 13 14</entry>
<entry __type="2s16">1232 -5433</entry>
<entry __type="2u16">345 3455</entry>
<entry __type="2s32">123123 -12672</entry>
<entry __type="2u32">5345 76575</entry>
<entry __type="2s64">5687 -5687</entry>
<entry __type="2s64">-6582345 52345</entry>
<entry __type="2u64">82345 52345</entry>
<entry __type="2u64">6582345 52345</entry>
<entry __type="2f">-32.000000 12.000000</entry>
<entry __type="2d">-32.000000 12.000000</entry>
<entry __type="2d">-32.000000 12.000000</entry>
<entry __type="3s8">28 127 -124</entry>
<entry __type="3u8">2 4 255</entry>
<entry __type="3s16">2842 12427 -14224</entry>
<entry __type="3u16">242 424 24255</entry>
<entry __type="3s32">2248 175627 -12244</entry>
<entry __type="3u32">24422 44242 25245</entry>
<entry __type="3s64">2898 10927 -12094</entry>
<entry __type="3u64">28908 8904 2955</entry>
<entry __type="3f">-32.000000 12.000000 3.000000</entry>
<entry __type="3d">-32.000000 12.000000 3.000000</entry>
<entry __type="4s8">24 28 127 -124</entry>
<entry __type="4u8">1 2 255 255</entry>
<entry __type="4s16">-34 -53 -23 34</entry>
<entry __type="4u16">123 123 154 654</entry>
<entry __type="4s32">12 132 656 -4565</entry>
<entry __type="4s32">12 132 -656 4565</entry>
<entry __type="4u32">12 132 656 4565</entry>
<entry __type="4u32">12 132 656 4565</entry>
<entry __type="4s64">-31231 5128741 -123222 129381231</entry>
<entry __type="4u64">31231 5128741 1280371203 129381231</entry>
<entry __type="4f">-32.000000 12.000000 3.000000 2.000000</entry>
<entry __type="4f">5.000000 52.000000 -12.000000 32.000000</entry>
<entry __type="4d">1.000000 3.000000 4.000000 -1.000000</entry>
<entry __type="vs8">1 3 5 7 12 16 46 76 93 123 0 -124 -123 -12 -42 -56</entry>
<entry __type="vu8">1 3 5 7 12 16 46 76 93 123 178 240 255 127 126 255</entry>
<entry __type="vs16">93 123 0 -124 -123 -12 -42 -56</entry>
<entry __type="vu16">16 46 76 93 123 178 240 255</entry>
<entry __type="bool">1</entry>
<entry __type="bool">1</entry>
<entry __type="2b">0 1</entry>
<entry __type="3b">1 0 1</entry>
<entry __type="4b">1 1 1 0</entry>
<entry __type="vb">1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0</entry>
</test>