diff --git a/README.md b/README.md index a250bd5..433b23a 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,18 @@ An encoder/decoder for Konami's binary XML format, used in some of their games. ### Setup: -`pip install bitarray` +`pip install bitarray lxml` -Python 2 only: +Additionally for Python 2: `pip install future` ```python In [1]: from kbinxml import KBinXML In [2]: text = KBinXML('Hello, world!') In [3]: text.to_binary() -Out[4]: '\xa0B\x80\x7f\x00\x00\x00\x08\x0b\x04\xdfM9\xfe\xff\x00\x00\x00\x00\x14\x00\x00\x00\x0eHello, world!\x00\x00\x00' +Out[4]: b'\xa0B\x80\x7f\x00\x00\x00\x08\x0b\x04\xdfM9\xfe\xff\x00\x00\x00\x00\x14\x00\x00\x00\x0eHello, world!\x00\x00\x00' In [5]: bin = KBinXML(Out[4]) In [6]: bin.to_text() -Out[7]: '\nHello, world!\n' +Out[7]: u'\nHello, world!\n' ``` \ No newline at end of file diff --git a/kbinxml.py b/kbinxml.py index 5b63429..c3caba1 100644 --- a/kbinxml.py +++ b/kbinxml.py @@ -1,11 +1,12 @@ # python 3 style, ints instead of b'' from builtins import bytes -from xml.dom import minidom from struct import calcsize import string import sys import operator +from io import BytesIO +import lxml.etree as etree from bytebuffer import ByteBuffer from sixbit import pack_sixbit, unpack_sixbit from format_ids import xml_formats, xml_types @@ -20,7 +21,7 @@ SIGNATURE = 0xA0 SIG_COMPRESSED = 0x42 SIG_UNCOMPRESSED = 0x45 -XML_ENCODING = 'UTF_8' +XML_ENCODING = 'UTF-8' BIN_ENCODING = 'SHIFT_JISX0213' # NOTE: all of these are their python codec names @@ -41,18 +42,22 @@ def debug_print(string): class KBinXML(): def __init__(self, input): - if isinstance(input, minidom.Document): + if isinstance(input, etree._Element): self.xml_doc = input + elif isinstance(input, etree._ElementTree): + self.xml_doc = input.getroot() elif KBinXML.is_binary_xml(input): self.from_binary(input) else: self.from_text(input) def to_text(self): - return self.xml_doc.toprettyxml(indent = " ", encoding = XML_ENCODING) + # we decode again because I want unicode, dammit + return etree.tostring(self.xml_doc, pretty_print=True, + encoding=XML_ENCODING, xml_declaration=True).decode(XML_ENCODING) def from_text(self, input): - self.xml_doc = minidom.parseString(input) + self.xml_doc = etree.parse(BytesIO(input)).getroot() @staticmethod def is_binary_xml(input): @@ -121,28 +126,30 @@ class KBinXML(): self.dataBuf.realign_writes() def _node_to_binary(self, node): - if node.nodeType == node.TEXT_NODE or node.nodeType == node.COMMENT_NODE: - return - nodeType = node.getAttribute('__type') + nodeType = node.attrib.get('__type') if not nodeType: - nodeType = 'void' + # typeless tags with text become string + if node.text is not None and len(node.text.strip()) > 0: + nodeType = 'str' + else: + nodeType = 'void' nodeId = xml_types[nodeType] isArray = 0 - count = node.getAttribute('__count') + count = node.attrib.get('__count') if count: count = int(count) isArray = 64 # bit position for array flag self.nodeBuf.append_u8(nodeId | isArray) - name = node.nodeName + name = node.tag pack_sixbit(name, self.nodeBuf) if nodeType != 'void': fmt = xml_formats[nodeId] - val = node.firstChild.nodeValue + val = node.text if fmt['name'] == 'bin': data = bytes(bytearray.fromhex(val)) elif fmt['name'] == 'str': @@ -161,14 +168,14 @@ class KBinXML(): self.data_append_aligned(data, fmt['type'], fmt['count']) # for test consistency and to be more faithful, sort the attrs - sorted_attrs = sorted(node.attributes.items(), key=operator.itemgetter(0)) + sorted_attrs = sorted(node.attrib.items(), key=operator.itemgetter(0)) for key, value in sorted_attrs: if key not in ['__type', '__size', '__count']: self.data_append_string(value) self.nodeBuf.append_u8(xml_types['attr']) pack_sixbit(key, self.nodeBuf) - for child in node.childNodes: + for child in node.iterchildren(tag=etree.Element): self._node_to_binary(child) # always has the isArray bit set @@ -188,8 +195,7 @@ class KBinXML(): self.dataByteBuf = ByteBuffer(self.dataBuf.data) self.dataWordBuf = ByteBuffer(self.dataBuf.data) - for child in self.xml_doc.childNodes: - self._node_to_binary(child) + self._node_to_binary(self.xml_doc) # always has the isArray bit set self.nodeBuf.append_u8(xml_types['endSection'] | 64) @@ -199,7 +205,7 @@ class KBinXML(): return bytes(header.data + self.nodeBuf.data + self.dataBuf.data) def from_binary(self, input): - self.xml_doc = minidom.Document() + self.xml_doc = etree.Element('root') node = self.xml_doc self.nodeBuf = ByteBuffer(input) @@ -249,10 +255,10 @@ class KBinXML(): if nodeType == xml_types['attr']: value = self.data_grab_string() - node.setAttribute(name, value) + node.attrib[name] = value elif nodeType == xml_types['nodeEnd']: - if node.parentNode: - node = node.parentNode + if node.getparent() is not None: + node = node.getparent() elif nodeType == xml_types['endSection']: nodesLeft = False elif nodeType not in xml_formats: @@ -263,14 +269,13 @@ class KBinXML(): if skip: continue - child = self.xml_doc.createElement(name) - node.appendChild(child) + child = etree.SubElement(node, name) node = child if nodeType == xml_types['nodeStart']: continue - node.setAttribute('__type', nodeFormat['name']) + node.attrib['__type'] = nodeFormat['name'] varCount = nodeFormat['count'] arrayCount = 1 @@ -279,7 +284,7 @@ class KBinXML(): isArray = True elif isArray: arrayCount = self.dataBuf.get_u32() // (calcsize(nodeFormat['type'] * varCount)) - node.setAttribute('__count', str(arrayCount)) + node.attrib['__count'] = str(arrayCount) totalCount = arrayCount * varCount if isArray: @@ -289,14 +294,17 @@ class KBinXML(): data = self.data_grab_aligned(nodeFormat['type'], totalCount) if nodeType == xml_types['binary']: - node.setAttribute('__size', str(totalCount)) + node.attrib['__size'] = str(totalCount) string = ''.join(('{0:02x}'.format(x) for x in data)) elif nodeType == xml_types['string']: string = bytes(data[:-1]).decode(self.encoding) else: string = ' '.join(map(nodeFormat.get('toStr', str), data)) - node.appendChild(self.xml_doc.createTextNode(string)) + node.text = string + + # because we need the 'real' root + self.xml_doc = self.xml_doc[0] if __name__ == '__main__': if len(sys.argv) != 2: @@ -308,6 +316,6 @@ if __name__ == '__main__': xml = KBinXML(input) if KBinXML.is_binary_xml(input): - stdout.write(xml.to_text()) + stdout.write(xml.to_text().encode('utf-8')) else: stdout.write(xml.to_binary()) diff --git a/test.py b/test.py new file mode 100644 index 0000000..6b1b8bb --- /dev/null +++ b/test.py @@ -0,0 +1,28 @@ +from kbinxml import KBinXML +# python 2/3 cross compat +from io import open + +with open('testcases.xml', 'rb') as f: + xml_in = f.read() +with open('testcases_out.xml', 'r', encoding='UTF-8') as f: + expected_xml = f.read() +with open('testcases_out.kbin', 'rb') as f: + expected_bin = f.read() + +k = KBinXML(xml_in) +kbin = k.to_binary() +if kbin != expected_bin: + with open('failed_test.bin', 'wb') as f: + f.write(kbin) + raise AssertionError('Binary output does not match, check failed_test.bin') +else: + print('XML -> Binary correct!') + +backwards = KBinXML(kbin) +btext = backwards.to_text() +if btext != expected_xml: + with open('failed_test.xml', 'w', encoding='UTF-8') as f: + f.write(btext) + raise AssertionError('XML putput does not match, check failed_test.xml') +else: + print('Binary -> XML correct!') diff --git a/testcases.xml b/testcases.xml index c8e2cf2..c7d7bb2 100644 --- a/testcases.xml +++ b/testcases.xml @@ -5,6 +5,8 @@ 127.0.0.1 192.168.0.1 Hello, world! + + ƒV@ƒCƒX@ƒ}ƒC@ƒƒCƒt 8 9 10 diff --git a/testcases_out.kbin b/testcases_out.kbin index e3ec604..31fdc48 100644 Binary files a/testcases_out.kbin and b/testcases_out.kbin differ diff --git a/testcases_out.xml b/testcases_out.xml index ce99fc8..0a65831 100644 --- a/testcases_out.xml +++ b/testcases_out.xml @@ -1,75 +1,76 @@ - + - 127.0.0.1 192.168.0.1 - Hello, world! - 8 9 10 - 12 - deadbeef - deadbe7a - 1 2 3 1 2 3 - - - 123 - 123 - 123 - 123 - 123 - 123 - 123 - 123 - f00d1337 - deadbeef - TEST STRING 2 - TEST STRING - 192.168.0.1 - 123.099998 - 123.099998 - 123.100000 - 123.100000 - 127 -23 - 232 12 13 14 - 1232 -5433 - 345 3455 - 123123 -12672 - 5345 76575 - 5687 -5687 - -6582345 52345 - 82345 52345 - 6582345 52345 - -32.000000 12.000000 - -32.000000 12.000000 - -32.000000 12.000000 - 28 127 -124 - 2 4 255 - 2842 12427 -14224 - 242 424 24255 - 2248 175627 -12244 - 24422 44242 25245 - 2898 10927 -12094 - 28908 8904 2955 - -32.000000 12.000000 3.000000 - -32.000000 12.000000 3.000000 - 24 28 127 -124 - 1 2 255 255 - -34 -53 -23 34 - 123 123 154 654 - 12 132 656 -4565 - 12 132 -656 4565 - 12 132 656 4565 - 12 132 656 4565 - -31231 5128741 -123222 129381231 - 31231 5128741 1280371203 129381231 - -32.000000 12.000000 3.000000 2.000000 - 5.000000 52.000000 -12.000000 32.000000 - 1.000000 3.000000 4.000000 -1.000000 - 1 3 5 7 12 16 46 76 93 123 0 -124 -123 -12 -42 -56 - 1 3 5 7 12 16 46 76 93 123 178 240 255 127 126 255 - 93 123 0 -124 -123 -12 -42 -56 - 16 46 76 93 123 178 240 255 - 1 - 1 - 0 1 - 1 0 1 - 1 1 1 0 - 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 127.0.0.1 192.168.0.1 + Hello, world! + ใ‚ทใ€€ใ‚คใ‚นใ€€ใƒžใ‚คใ€€ใƒฏใ‚คใƒ• + 8 9 10 + 12 + deadbeef + deadbe7a + 1 2 3 1 2 3 + + + 123 + 123 + 123 + 123 + 123 + 123 + 123 + 123 + f00d1337 + deadbeef + TEST STRING 2 + TEST STRING + 192.168.0.1 + 123.099998 + 123.099998 + 123.100000 + 123.100000 + 127 -23 + 232 12 13 14 + 1232 -5433 + 345 3455 + 123123 -12672 + 5345 76575 + 5687 -5687 + -6582345 52345 + 82345 52345 + 6582345 52345 + -32.000000 12.000000 + -32.000000 12.000000 + -32.000000 12.000000 + 28 127 -124 + 2 4 255 + 2842 12427 -14224 + 242 424 24255 + 2248 175627 -12244 + 24422 44242 25245 + 2898 10927 -12094 + 28908 8904 2955 + -32.000000 12.000000 3.000000 + -32.000000 12.000000 3.000000 + 24 28 127 -124 + 1 2 255 255 + -34 -53 -23 34 + 123 123 154 654 + 12 132 656 -4565 + 12 132 -656 4565 + 12 132 656 4565 + 12 132 656 4565 + -31231 5128741 -123222 129381231 + 31231 5128741 1280371203 129381231 + -32.000000 12.000000 3.000000 2.000000 + 5.000000 52.000000 -12.000000 32.000000 + 1.000000 3.000000 4.000000 -1.000000 + 1 3 5 7 12 16 46 76 93 123 0 -124 -123 -12 -42 -56 + 1 3 5 7 12 16 46 76 93 123 178 240 255 127 126 255 + 93 123 0 -124 -123 -12 -42 -56 + 16 46 76 93 123 178 240 255 + 1 + 1 + 0 1 + 1 0 1 + 1 1 1 0 + 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0