Leahnaya/butterfly
1
0
Fork 0
mirror of https://github.com/skogaby/butterfly.git synced 2026-03-21 18:04:24 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use a Kotlin kbinxml implementation instead of shelling out to Python for the conversions

Browse Source
This commit is contained in:
skogaby 2019-01-18 15:28:27 -06:00
parent b11f658e4d
commit b0d510af16
30 changed files with 1033 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -11,7 +11,6 @@
.mtj.tmp/
# Package Files #
*.jar
*.war
*.nar
*.ear
@ -33,6 +32,7 @@ gradle-app.setting
.gradletasknamecache
/bin/
.DS_Store
*.iml
# vscode stuff
.project

28
build.gradle
View File

@ -2,6 +2,7 @@ group 'com.buttongames'
version '1.0-SNAPSHOT'
apply plugin: 'java'
apply plugin: 'kotlin'
apply plugin: 'application'
sourceCompatibility = 1.8
@ -13,6 +14,8 @@ repositories {
}
dependencies {
implementation fileTree(dir: 'lib', include: ['*.jar'])
testCompile group: 'junit', name: 'junit', version: '4.12'
// Spark, core HTTP server provider
@ -37,6 +40,9 @@ dependencies {
compile group: 'org.hibernate', name: 'hibernate-java8', version: '5.4.0.Final'
compile group: 'org.springframework', name: 'spring-orm', version: '5.1.3.RELEASE'
compile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.25.2'
// Kotlin, for kbinxml support
compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
}
jar {
@ -51,4 +57,26 @@ jar {
it.isDirectory() ? it : zipTree(it)
}
}
}
buildscript {
ext.kotlin_version = '1.3.11'
repositories {
mavenCentral()
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
}
}
compileKotlin {
kotlinOptions {
jvmTarget = "1.8"
}
}
compileTestKotlin {
kotlinOptions {
jvmTarget = "1.8"
}
}

BIN
lib/xom-1.3.0-SNAPSHOT.jar Normal file
View File

Binary file not shown.

12
src/main/java/com/buttongames/butterfly/http/ButterflyHttpServer.java
View File

@ -27,8 +27,8 @@ import com.buttongames.butterfly.http.handlers.impl.TaxRequestHandler;
import com.buttongames.butterfly.model.ButterflyUser;
import com.buttongames.butterfly.model.Machine;
import com.buttongames.butterfly.util.PropertyNames;
import com.buttongames.butterfly.xml.BinaryXmlUtils;
import com.buttongames.butterfly.xml.XmlUtils;
import com.buttongames.butterfly.xml.kbinxml.PublicKt;
import com.google.common.collect.ImmutableSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -299,13 +299,15 @@ public class ButterflyHttpServer {
}
// convert the body to plaintext XML if it's binary XML
if (BinaryXmlUtils.isBinaryXML(reqBody)) {
reqBody = BinaryXmlUtils.binaryToXml(reqBody);
Element rootNode = null;
if (XmlUtils.isBinaryXML(reqBody)) {
rootNode = XmlUtils.stringToXmlFile(PublicKt.kbinDecodeToString(reqBody));
} else {
rootNode = XmlUtils.byteArrayToXmlFile(reqBody);
}
// read the request body into an XML document
Element rootNode = XmlUtils.byteArrayToXmlFile(reqBody);
if (rootNode == null ||
!rootNode.getNodeName().equals("call")) {
throw new InvalidRequestException();

9
src/main/java/com/buttongames/butterfly/http/handlers/BaseRequestHandler.java
View File

@ -1,8 +1,8 @@
package com.buttongames.butterfly.http.handlers;
import com.buttongames.butterfly.compression.Lz77;
import com.buttongames.butterfly.encryption.Rc4;
import com.buttongames.butterfly.xml.BinaryXmlUtils;
import com.buttongames.butterfly.xml.XmlUtils;
import com.buttongames.butterfly.xml.kbinxml.PublicKt;
import com.google.common.net.MediaType;
import com.jamesmurty.utils.BaseXMLBuilder;
import org.apache.logging.log4j.LogManager;
@ -27,7 +27,6 @@ import java.nio.file.Paths;
import static com.buttongames.butterfly.util.Constants.COMPRESSION_HEADER;
import static com.buttongames.butterfly.util.Constants.CRYPT_KEY_HEADER;
import static com.buttongames.butterfly.util.Constants.LZ77_COMPRESSION;
/**
* Base request handler that the others inherit from.
@ -106,8 +105,8 @@ public abstract class BaseRequestHandler {
response.header("Connection", "keep-alive");
// convert them to binary XML
if (!BinaryXmlUtils.isBinaryXML(respBytes)) {
respBytes = BinaryXmlUtils.xmlToBinary(respBytes);
if (!XmlUtils.isBinaryXML(respBytes)) {
respBytes = PublicKt.kbinEncode(new String(respBytes));
}
// TODO: FIX THIS SHIT

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/CardManageRequestHandler.java
View File

@ -11,7 +11,7 @@ import com.buttongames.butterfly.model.CardType;
import com.buttongames.butterfly.util.CardIdUtils;
import com.buttongames.butterfly.util.StringUtils;
import com.buttongames.butterfly.xml.XmlUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/EventLogRequestHandler.java
View File

@ -5,7 +5,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.model.ddr16.GameplayEventLog;
import com.buttongames.butterfly.util.TimeUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import com.buttongames.butterfly.xml.XmlUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/FacilityRequestHandler.java
View File

@ -6,7 +6,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.model.ddr16.Shop;
import com.buttongames.butterfly.util.StringUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/MessageRequestHandler.java
View File

@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl;
import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.util.PropertyNames;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/PackageRequestHandler.java
View File

@ -2,7 +2,7 @@ package com.buttongames.butterfly.http.handlers.impl;
import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbEventRequestHandler.java
View File

@ -5,7 +5,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.model.ddr16.PcbEventLog;
import com.buttongames.butterfly.util.TimeUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import com.buttongames.butterfly.xml.XmlUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbTrackerRequestHandler.java
View File

@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl;
import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.util.PropertyNames;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/PlayerDataRequestHandler.java
View File

@ -25,7 +25,7 @@ import com.buttongames.butterfly.model.ddr16.options.SpeedOption;
import com.buttongames.butterfly.model.ddr16.options.StepZoneOption;
import com.buttongames.butterfly.model.ddr16.options.TurnOption;
import com.buttongames.butterfly.util.StringUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import com.buttongames.butterfly.xml.XmlUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/ServicesRequestHandler.java
View File

@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl;
import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.util.PropertyNames;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import com.google.common.collect.ImmutableMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/SystemRequestHandler.java
View File

@ -4,7 +4,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.util.CardIdUtils;
import com.buttongames.butterfly.xml.XmlUtils;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;

2
src/main/java/com/buttongames/butterfly/http/handlers/impl/TaxRequestHandler.java
View File

@ -6,7 +6,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException;
import com.buttongames.butterfly.http.handlers.BaseRequestHandler;
import com.buttongames.butterfly.model.Machine;
import com.buttongames.butterfly.model.UserPhases;
import com.buttongames.butterfly.xml.builder.KXmlBuilder;
import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

101
src/main/java/com/buttongames/butterfly/xml/BinaryXmlUtils.java
View File

@ -1,101 +0,0 @@
package com.buttongames.butterfly.xml;
import com.google.common.io.ByteStreams;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
/**
* Class to help translating between binary and plaintext XML. Right now it's a very dumb
* class and just uses mon's Python implementation until I can make a native Java one. This does
* depend on you having done a <code>pip install kbinxml</code> to make <code>kbinxml</code>
* a valid command.
* See: https://github.com/mon/kbinxml
* @author skogaby (skogabyskogaby@gmail.com)
*/
public class BinaryXmlUtils {
/**
* First bytes of a plaintext XML response, so we know if an array needs to be converted
* to/from binary XML.
*/
private static final byte[] XML_PREFIX = "<?xml".getBytes(StandardCharsets.UTF_8);
/**
* Converts the input to plaintext XML from binary.
* @param input
* @return
* @throws IOException
*/
public static byte[] binaryToXml(final byte[] input) {
try {
final ProcessBuilder builder = new ProcessBuilder("python",
"-c",
"import sys; " +
"from kbinxml import KBinXML; " +
"the_bytes = sys.stdin.buffer.read(); "+
"print(KBinXML(the_bytes).to_text())");
final Process process = builder.start();
final OutputStream stdin = process.getOutputStream();
final InputStream stdout = process.getInputStream();
stdin.write(input, 0, input.length);
stdin.flush();
stdin.close();
return ByteStreams.toByteArray(stdout);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* Converts the input to binary XML from plaintext XML.
* @param input
* @return
* @throws IOException
*/
public static byte[] xmlToBinary(final byte[] input) {
try {
final ProcessBuilder builder = new ProcessBuilder("python",
"-c",
"import sys; " +
"from kbinxml import KBinXML; " +
"the_bytes = sys.stdin.buffer.read(); " +
"sys.stdout.buffer.write(KBinXML(the_bytes).to_binary())");
final Process process = builder.start();
final OutputStream stdin = process.getOutputStream();
final InputStream stdout = process.getInputStream();
stdin.write(input, 0, input.length);
stdin.flush();
stdin.close();
return ByteStreams.toByteArray(stdout);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* Says whether or not the input is binary XML.
* @param input
* @return
*/
public static boolean isBinaryXML(final byte[] input) {
boolean isBinary = false;
for (int i = 0; i < XML_PREFIX.length; i++) {
if (input[i] != XML_PREFIX[i]) {
isBinary = true;
break;
}
}
return isBinary;
}
}

47
src/main/java/com/buttongames/butterfly/xml/XmlUtils.java
View File

@ -23,6 +23,30 @@ public class XmlUtils {
private static final XPath XPATH = XPathFactory.newInstance().newXPath();
/**
* First bytes of a plaintext XML response, so we know if an array needs to be converted
* to/from binary XML.
*/
private static final byte[] XML_PREFIX = "<?xml".getBytes();
/**
* Says whether or not the input is binary XML.
* @param input
* @return
*/
public static boolean isBinaryXML(final byte[] input) {
boolean isBinary = false;
for (int i = 0; i < XML_PREFIX.length; i++) {
if (input[i] != XML_PREFIX[i]) {
isBinary = true;
break;
}
}
return isBinary;
}
/**
* Scrubs empty nodes from a document so we don't accidentally read them.
* @param node The root node of the document to clean.
@ -49,6 +73,7 @@ public class XmlUtils {
}
}
}
/**
* Reads the given byte[] into an Element that represents the root node of the XML body.
* @param body
@ -71,6 +96,28 @@ public class XmlUtils {
}
}
/**
* Reads the given string into an Element that represents the root node of the XML body.
* @param body
* @return
* @throws ParserConfigurationException
* @throws IOException
* @throws SAXException
*/
public static Element stringToXmlFile(final String body) {
try {
final DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
final DocumentBuilder builder = builderFactory.newDocumentBuilder();
final Document reqDocument = builder.parse(new ByteArrayInputStream(body.getBytes()));
XmlUtils.clean(reqDocument);
return reqDocument.getDocumentElement();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* Reads the String value at the given XPath expression from the given document.
* @param doc

101
src/main/java/com/buttongames/butterfly/xml/kbinxml/ByteConv.kt Normal file
View File

@ -0,0 +1,101 @@
package com.buttongames.butterfly.xml.kbinxml
import com.buttongames.butterfly.xml.kbinxml.ByteConv.E.longToBytes
import java.lang.Double
import java.lang.Float
import java.math.BigInteger
import java.nio.ByteBuffer
import kotlin.experimental.and
internal class ByteConv {
companion object E {
fun bytesToLong(array: ByteArray, numBytes: Int, littleEndian: Boolean = false): Long {
var bytes = array
if (!littleEndian) {
bytes = array.reversedArray()
}
val actualArray = bytes.unsigned()
var result: Long = 0
for (i in 0 until numBytes) {
result = result or (actualArray[i].toLong() shl (i * 8))
}
return result
}
fun longToBytes(input: Long, numBytes: Int): ByteArray {
val result = ByteArray(numBytes)
for (i in 0 until numBytes) {
result[numBytes - i - 1] = (input shr (i * 8)).toByte()
}
return result
}
/*fun binToString(array: ByteArray): String {
val sb = StringBuilder()
for (e in array) {
var current = e.toUInt().toInt().toString(16)
if (current.length == 1) current = "0$current"
sb.append(current)
}
return sb.toString()
}*/
private val hexArray = "0123456789abcdef".toCharArray()
fun binToString(bytes: ByteArray): String {
val hexChars = CharArray(bytes.size * 2)
for (j in bytes.indices) {
val v = bytes[j].posInt()
hexChars[j * 2] = hexArray[v.ushr(4)]
hexChars[j * 2 + 1] = hexArray[v and 0x0F]
}
return String(hexChars)
}
fun stringToBin(s: String): ByteArray {
val len = s.length
val data = ByteArray(len / 2)
var i = 0
while (i < len) {
data[i / 2] = ((Character.digit(s[i], 16) shl 4) + Character.digit(s[i + 1], 16)).toByte()
i += 2
}
return data
}
fun ipToString(array: ByteArray) = array.joinToString(".") { it.toUByte().toString() }
fun stringToIp(string: String) = string.split(".").map { it.toUByte().toByte() }.toByteArray()
fun floatToString(array: ByteArray) = String.format("%.6f", ByteBuffer.wrap(array).float)
fun doubleToString(array: ByteArray) = String.format("%.6f", ByteBuffer.wrap(array).double)
fun stringToFloat(string: String) = Float.floatToRawIntBits(Float.parseFloat(string)).toByteArray()
fun stringToDouble(string: String) = Double.doubleToRawLongBits(Double.parseDouble(string)).toByteArray()
fun boolToString(array: ByteArray) = (array[0] and 1).toString()
fun stringToBool(string: String) = byteArrayOf(if (string == "1") 1 else 0)
}
}
internal fun ByteArray.toByte(): Byte = this[0]
internal fun ByteArray.toShort(): Short = ByteConv.bytesToLong(this, 2).toShort()
internal fun ByteArray.toInt(): Int = ByteConv.bytesToLong(this, 4).toInt()
internal fun ByteArray.toLong(): Long = ByteConv.bytesToLong(this, 8)
internal fun ByteArray.toUByte(): UByte = this[0].toUByte()
internal fun ByteArray.toUShort(): UShort = this.toShort().toUShort()
internal fun ByteArray.toUInt(): UInt = this.toInt().toUInt()
internal fun ByteArray.toULong(): ULong = this.toLong().toULong()
internal fun Short.toByteArray() = longToBytes(this.toLong(), 2)
internal fun Int.toByteArray() = longToBytes(this.toLong(), 4)
internal fun Long.toByteArray() = longToBytes(this, 8)
internal fun String.toByteA() = byteArrayOf(BigInteger(this).toByte())
internal fun String.toShortBytes() = BigInteger(this).toShort().toByteArray()
internal fun String.toIntBytes() = BigInteger(this).toInt().toByteArray()
internal fun String.toLongBytes() = BigInteger(this).toLong().toByteArray()
internal inline fun String.toUByteA() = this.toByteA()
internal inline fun String.toUShortBytes() = this.toShortBytes()
internal inline fun String.toUIntBytes() = this.toIntBytes()
internal inline fun String.toULongBytes() = this.toLongBytes()

151
src/main/java/com/buttongames/butterfly/xml/kbinxml/Constants.kt Normal file
View File

@ -0,0 +1,151 @@
package com.buttongames.butterfly.xml.kbinxml
internal class Constants {
companion object {
val encodings = arrayOf("SHIFT_JIS", "ASCII", "ISO-8859-1", "EUC-JP", "SHIFT_JIS", "UTF-8")
val encodingsReverse: Map<String, Int>
init {
val m1 = mutableMapOf<String, Int>()
for (i in 1 until encodings.size) {
m1[encodings[i]] = i
}
encodingsReverse = m1
}
}
}
internal enum class ControlTypes {
NodeStart,
Attribute,
NodeEnd,
FileEnd
}
internal val ControlTypeMap = mapOf(
1 to ControlTypes.NodeStart,
46 to ControlTypes.Attribute,
190 to ControlTypes.NodeEnd,
191 to ControlTypes.FileEnd
//254 to ControlTypes.NodeEnd,
//255 to ControlTypes.FileEnd
)
internal class KbinConverter(val fromString: (String) -> ByteArray, val toString: (ByteArray) -> String)
internal class KbinType(var names: List<String>, val size: Int, private val handler: KbinConverter, val count: Int = 1) {
constructor(name: String, size: Int, handler: KbinConverter, count: Int = 1) : this(mutableListOf(name), size, handler, count)
val name: String
get() = names[0]
fun alias(alias: String): KbinType {
(names as MutableList<String>) += alias
return this
}
fun rename(name: String): KbinType {
val mutable = (names as MutableList<String>)
mutable.clear()
mutable.add(name)
return this
}
fun fromString(string: String): ByteArray = if (string.isEmpty()) byteArrayOf() else handler.fromString(string)
fun toString(bytes: ByteArray): String = if (bytes.isEmpty()) "" else handler.toString(bytes)
}
internal operator fun Int.times(t: KbinType): KbinType {
val newNames = t.names.map { this.toString() + it }
val newSize = t.size * this
fun newToString(input: ByteArray) =
input.asIterable().chunked(t.size).joinToString(" ") { t.toString(it.toByteArray()) }
fun newFromString(input: String) =
input.split(" ").flatMap { t.fromString(it).asIterable() }.toByteArray()
return KbinType(newNames, newSize, KbinConverter(::newFromString, ::newToString), this * t.count)
}
internal class Types {
companion object {
val s8 = KbinType("s8", 1, Converters.s8)
val u8 = KbinType("u8", 1, Converters.u8)
val s16 = KbinType("s16", 2, Converters.s16)
val u16 = KbinType("u16", 2, Converters.u16)
val s32 = KbinType("s32", 4, Converters.s32)
val u32 = KbinType("u32", 4, Converters.u32)
val s64 = KbinType("s64", 8, Converters.s64)
val u64 = KbinType("u64", 8, Converters.u64)
val bin = KbinType(listOf("bin", "binary"), 1, Converters.stub)
val strStub = KbinType(listOf("str", "string"), 1, Converters.stub)
val ip4 = KbinType("ip4", 4, Converters.ip4)
val time = KbinType("time", 4, Converters.u32)
val float = KbinType(listOf("float", "f"), 4, Converters.float)
val double = KbinType(listOf("double", "d"), 8, Converters.double)
val bool = KbinType(listOf("bool", "b"), 1, Converters.bool)
}
}
internal val kbinTypeMap = with(Types) {
mapOf(
2 to s8,
3 to u8,
4 to s16,
5 to u16,
6 to s32,
7 to u32,
8 to s64,
9 to u64,
10 to bin,
11 to strStub,
12 to ip4,
13 to time,
14 to float,
15 to double,
16 to 2 * s8,
17 to 2 * u8,
18 to 2 * s16,
19 to 2 * u16,
20 to 2 * s32,
21 to 2 * u32,
22 to (2 * s64).alias("vs64"),
23 to (2 * u64).alias("vu64"),
24 to (2 * float).rename("2f"),
25 to (2 * double).rename("2d").alias("vd"),
26 to 3 * s8,
27 to 3 * u8,
28 to 3 * s16,
29 to 3 * u16,
30 to 3 * s32,
31 to 3 * u32,
32 to 3 * s64,
33 to 3 * u64,
34 to (3 * float).rename("3f"),
35 to (3 * double).rename("3d"),
36 to 4 * s8,
37 to 4 * u8,
38 to 4 * s16,
39 to 4 * u16,
40 to (4 * s32).alias("vs32"),
41 to (4 * u32).alias("vu32"),
42 to 4 * s64,
43 to 4 * u64,
44 to (4 * float).rename("4f").alias("vf"),
45 to (4 * double).rename("4d"),
48 to (16 * s8).rename("vs8"),
49 to (16 * u8).rename("vu8"),
50 to (8 * s16).rename("vs16"),
51 to (8 * u16).rename("vu16"),
52 to bool,
53 to (2 * bool).rename("2b"),
54 to (3 * bool).rename("3b"),
55 to (4 * bool).rename("4b"),
56 to (16 * bool).rename("vb")
)
}
internal var reverseKbinTypeMap: Map<String, Int> = kbinTypeMap.entries.associateBy({ it.value.name }) { it.key }

2
src/main/java/com/buttongames/butterfly/xml/builder/KXmlBuilder.java → src/main/java/com/buttongames/butterfly/xml/kbinxml/KXmlBuilder.java
View File

@ -1,4 +1,4 @@
package com.buttongames.butterfly.xml.builder;
package com.buttongames.butterfly.xml.kbinxml;
/**
* CONTEXT: I (skogaby) wanted to extend XMLBuilder2 from java-xmlbuilder to provide convenience

23
src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinConverters.kt Normal file
View File

@ -0,0 +1,23 @@
package com.buttongames.butterfly.xml.kbinxml
internal class Converters {
companion object {
val s8 = KbinConverter({ it.toByteA() }) { it.toByte().toString() }
val u8 = KbinConverter({ it.toUByteA() }) { it.toUByte().toString() }
val s16 = KbinConverter({ it.toShortBytes() }) { it.toShort().toString() }
val u16 = KbinConverter({ it.toUShortBytes() }) { it.toUShort().toString() }
val s32 = KbinConverter({ it.toIntBytes() }) { it.toInt().toString() }
val u32 = KbinConverter({ it.toUIntBytes() }) { it.toUInt().toString() }
val s64 = KbinConverter({ it.toLongBytes() }) { it.toLong().toString() }
val u64 = KbinConverter({ it.toULongBytes() }) { it.toULong().toString() }
// val bin = KbinConverter(ByteConv.E::stringToBin, ByteConv.E::binToString)
val stub = KbinConverter({ "STUB".toByteArray() }, { "STUB" }) // scary
val ip4 = KbinConverter(ByteConv.E::stringToIp, ByteConv.E::ipToString)
val float = KbinConverter(ByteConv.E::stringToFloat, ByteConv.E::floatToString)
val double = KbinConverter(ByteConv.E::stringToDouble, ByteConv.E::doubleToString)
val bool = KbinConverter(ByteConv.E::stringToBool, ByteConv.E::boolToString)
}
}

161
src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinDataBuffer.kt Normal file
View File

@ -0,0 +1,161 @@
package com.buttongames.butterfly.xml.kbinxml
import java.nio.charset.Charset
internal class KbinDataBuffer(bytes: ByteArray, val encoding: Charset) {
constructor(encoding: Charset) : this(byteArrayOf(), encoding)
private val data = bytes.toMutableList()
val size: Int
get() = data.size
private var pos8 = 0
private var pos16 = 0
private var pos32 = 0
fun readBytes(num: Int): ByteArray {
val result: ByteArray
val debug: String
when {
num == 0 -> return byteArrayOf()
num == 1 -> {
result = byteArrayOf(data[pos8])
debug = "$pos8"
}
num == 2 -> {
result = data.slice(pos16 until pos16 + 2).toByteArray()
debug = "$pos16 - ${pos16 + 2 - 1}"
}
num >= 3 -> {
result = data.slice(pos32 until pos32 + num).toByteArray()
debug = "$pos32 - ${pos32 + num - 1}"
}
else -> throw KbinException("Invalid read of $num bytes")
}
realign(num)
// println("Read bytes $debug")
return result
}
private fun realign(bytesRead: Int) {
fun pos8Follows() = pos8 % 4 == 0
fun pos16Follows() = pos16 % 4 == 0
if (bytesRead == 1) {
if (pos8Follows()) {
pos32 += 4
}
pos8++
}
if (bytesRead == 2) {
if (pos16Follows()) {
pos32 += 4
}
pos16 += 2
}
if (bytesRead >= 3) {
var newNum = bytesRead
if (newNum % 4 != 0) newNum += 4 - (newNum % 4)
pos32 += newNum
}
if (pos8Follows()) {
pos8 = pos32
}
if (pos16Follows()) {
pos16 = pos32
}
//println("Index 4: $pos32")
//println("Index 2: $pos16")
//println("Index 1: $pos8")
}
fun realign4Byte(num: Int) {
realign(num + (if (num % 4 != 0) {
4 - (num % 4)
} else 0))
}
fun reset() {
pos8 = 0; pos16 = 0; pos32 = 0
}
fun readU8(): UByte {
val result = readBytes(1)[0].toUByte()
return result
}
fun readU16(): UShort {
val result = readBytes(2)
return result.toUShort()
}
fun readU32(): UInt {
val result = readBytes(4)
return result.toUInt()
}
fun readFrom4Byte(num: Int): ByteArray {
if (num == 0) return byteArrayOf()
val read = data.slice(pos32 until pos32 + num)
realign4Byte(num)
return read.toByteArray()
}
fun readString(length: Int): String {
var readBytes = readFrom4Byte(length)
if (readBytes.last() == 0x00.toByte()) { // null bytes are scary
readBytes = readBytes.sliceArray(0 until readBytes.lastIndex)
}
return readBytes.toString(encoding)
}
fun writeBytes(bytes: ByteArray) {
val length = bytes.size
when {
length == 0 -> return
length == 1 -> {
data.setOrAddAll(pos8, bytes)
}
length == 2 -> {
data.setOrAddAll(pos16, bytes)
}
length >= 3 -> {
data.setOrAddAll(pos32, bytes)
}
else -> {
throw KbinException("Invalid write of $length bytes")
}
}
realign(length)
}
fun writeU8(value: UByte) {
writeBytes(byteArrayOf(value.toByte()))
}
fun writeU32(value: UInt) {
writeBytes(value.toInt().toByteArray())
}
fun writeTo4Byte(bytes: ByteArray) {
data.setOrAddAll(pos32, bytes)
realign4Byte(bytes.size)
}
fun writeString(string: String) {
var bytes = string.toByteArray(encoding) + 0 // null byte
writeU32(bytes.size.toUInt())
writeTo4Byte(bytes)
}
fun getContent(): ByteArray {
return data.toByteArray()
}
fun pad() {
while (data.size % 4 != 0) data.add(0)
}
}

67
src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinNodeBuffer.kt Normal file
View File

@ -0,0 +1,67 @@
package com.buttongames.butterfly.xml.kbinxml
import java.nio.charset.Charset
import kotlin.math.roundToInt
internal class KbinNodeBuffer(bytes: ByteArray, val compressed: Boolean, val encoding: Charset) {
constructor (compressed: Boolean, encoding: Charset) : this(byteArrayOf(), compressed, encoding)
private val data = bytes.toMutableList()
private var index = 0
val size: Int
get() = data.size
fun readU8() = readBytes(1)[0].toUByte()
fun readBytes(num: Int): ByteArray {
val result = data.slice(index until index + num).toByteArray()
index += num
return result
}
fun reset() {
index = 0
}
fun readString(): String {
val length = readU8().toInt()
if (compressed) {
val toRead = Math.ceil(length * 6 / 8.0).roundToInt()
val nameBytes = readBytes(toRead)
return Sixbit.decode(nameBytes, length)
} else {
val readBytes = readBytes((length and 64.inv()) + 1)
return readBytes.toString(encoding)
}
}
fun writeBytes(bytes: ByteArray) {
data.setOrAddAll(index, bytes)
index += bytes.size
}
fun writeU8(byte: UByte) {
writeBytes(byteArrayOf(byte.toByte()))
}
fun writeString(string: String) {
val bytes: ByteArray
if (compressed) {
bytes = Sixbit.encode(string)
writeU8(string.length.toUByte())
} else {
bytes = string.toByteArray(encoding)
writeU8(((bytes.size - 1) or (1 shl 6)).toUByte())
}
writeBytes(bytes)
}
fun getContent(): ByteArray {
return data.toByteArray()
}
fun pad() {
while (data.size % 4 != 0) data.add(0)
}
}

133
src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinReader.kt Normal file
View File

@ -0,0 +1,133 @@
package com.buttongames.butterfly.xml.kbinxml
import nu.xom.Document
import nu.xom.Element
import com.buttongames.butterfly.xml.kbinxml.ControlTypes.*
import java.nio.charset.Charset
import java.util.*
import kotlin.experimental.inv
internal class KbinReader(data: ByteArray) {
init {
if (data[0] != 0xA0.toByte()) {
throw KbinException("First byte must be 0xA0")
}
}
val compressed = when (data[1].toInt()) {
0x42 -> true
0x45 -> false
else -> throw KbinException("Second byte of data must be 0x42 or 0x45")
}
val encoding = when (
val tmp = data[2].posInt() shr 5) {
in 0..5 -> Charset.forName(Constants.encodings[tmp])
else -> throw KbinException("Third byte does not match any encoding")
}
init {
if (data[2] != data[3].inv()) throw KbinException("Fourth byte must be inverse of third")
}
private val nodeBuffer: KbinNodeBuffer
private val dataBuffer: KbinDataBuffer
init {
val nodeLength = data.slice(4 until 8).toByteArray().toUInt().toInt()
nodeBuffer = KbinNodeBuffer(data.slice(8 until (8 + nodeLength)).toByteArray(), compressed, encoding)
val dataStart = 12 + nodeLength
val dataLength = data.slice(dataStart - 4 until dataStart).toByteArray().toUInt().toInt()
dataBuffer = KbinDataBuffer(data.slice(dataStart until (dataStart + dataLength)).toByteArray(), encoding)
}
fun getXml(): Document {
nodeBuffer.reset(); dataBuffer.reset()
var currentNode: Element? = null
val nodeStack = ArrayDeque<Element>()
var end = false
while (!end) {
val current = nodeBuffer.readU8().toInt()
val actual = current and (1 shl 6).inv()
val controlType = ControlTypeMap[actual]
val valueType = kbinTypeMap[actual]
if (controlType != null)
when (controlType) {
NodeStart -> {
val name = nodeBuffer.readString()
val newNode = Element(name)
if (currentNode != null) {
currentNode.appendChild(newNode)
nodeStack.push(currentNode)
}
currentNode = newNode
}
NodeEnd -> {
if (nodeStack.size > 0) {
//println("Popping Node ${nodeStack.peek().localName}")
//currentNode?.sortAttributes()
currentNode = nodeStack.pop()
}
}
Attribute -> {
val name = nodeBuffer.readString()
val valueLength = dataBuffer.readU32().toInt()
val value = dataBuffer.readString(valueLength)
//println("Got attribute $name with value $value")
currentNode!!.addAttribute(name, value)
}
FileEnd -> {
if (nodeStack.size == 0) {
end = true
} else throw KbinException("Byte indicates end of file, but parsing is not done")
}
}
else if (valueType != null) {
val valueName = valueType.name
val isArray = (((current shr 6) and 1) == 1) or (valueName in listOf("bin", "str"))
val nodeName = nodeBuffer.readString()
val arraySize = if (isArray) dataBuffer.readU32().toInt() else valueType.size
nodeStack.push(currentNode)
val newNode = Element(nodeName)
newNode.addAttribute("__type", valueName)
currentNode!!.appendChild(newNode)
currentNode = newNode
val numElements = arraySize / valueType.size
when (valueName) {
"bin" -> {
currentNode.addAttribute("__size", arraySize.toString())
val bytes = dataBuffer.readFrom4Byte(arraySize)
currentNode.text = ByteConv.binToString(bytes)
}
"str" -> {
currentNode.text = dataBuffer.readString(arraySize)
}
else -> {
if (isArray) currentNode.addAttribute("__count", numElements.toString())
val byteList = dataBuffer.readBytes(arraySize)
val stringList = mutableListOf<String>()
for (i in 0 until numElements) {
val bytes = byteList.sliceArray(i * valueType.size until (i + 1) * valueType.size)
stringList.add(valueType.toString(bytes))
}
currentNode.text = stringList.joinToString(separator = " ")
}
}
} else if (current != 1) {
throw KbinException("Unsupported node type with ID $actual")
}
}
return Document(currentNode)
}
}

98
src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinWriter.kt Normal file
View File

@ -0,0 +1,98 @@
package com.buttongames.butterfly.xml.kbinxml
import nu.xom.Document
import nu.xom.Element
import java.io.ByteArrayOutputStream
import java.nio.charset.Charset
import kotlin.experimental.inv
internal class KbinWriter(val xml: Document, val encoding: String = "SHIFT_JIS", val compressed: Boolean = true) {
private val header = ByteArray(4)
val charset = Charset.forName(encoding)
fun getKbin(): ByteArray {
val dataBuffer = KbinDataBuffer(charset)
val nodeBuffer = KbinNodeBuffer(compressed, charset)
header[0] = 0xa0u.toByte()
header[1] = (if (compressed) 0x42u else 0x45u).toByte()
header[2] = (Constants.encodingsReverse[encoding]!! shl 5).toByte()
header[3] = header[2].inv()
nodeRecurse(xml.rootElement, dataBuffer, nodeBuffer)
nodeBuffer.writeU8(255u)
nodeBuffer.pad()
dataBuffer.pad()
val output = ByteArrayOutputStream()
output.write(header)
output.write(nodeBuffer.size.toByteArray())
output.write(nodeBuffer.getContent())
output.write(dataBuffer.size.toByteArray())
output.write(dataBuffer.getContent())
return output.toByteArray()
}
private fun nodeRecurse(e: Element, dataBuffer: KbinDataBuffer, nodeBuffer: KbinNodeBuffer) {
val typeName = e.getAttribute("__type")?.value
val typeId = reverseKbinTypeMap[typeName]
if (typeName != null && typeId == null) throw KbinException("Type $typeName is not supported")
val type = kbinTypeMap[typeId]
val count = (e.getAttribute("__count")?.value ?: e.getAttribute("__size")?.value)?.toInt()
val isArray = (count != null) && type?.name !in listOf("bin", "str")
if (typeId == null) {
nodeBuffer.writeU8(1u)
} else {
var toWrite = typeId.toUByte()
if (isArray) {
toWrite = toWrite or ((1 shl 6).toUByte())
}
nodeBuffer.writeU8(toWrite)
}
nodeBuffer.writeString(e.localName)
if (type != null) {
if (type.name == "bin") {
if (count != null) {
dataBuffer.writeU32(count.toUInt())
}
val toWrite = ByteConv.stringToBin(e.text)
dataBuffer.writeTo4Byte(toWrite)
} else if (type.name == "str") {
dataBuffer.writeString(e.text)
} else {
if (count != null) {
dataBuffer.writeU32((count * type.size).toUInt())
}
val split = e.text.splitAndJoin(type.count)
val toWrite = split.flatMap { type.fromString(it).asIterable() }.toByteArray()
dataBuffer.writeBytes(toWrite)
}
}
/*for (a in e) {
val name = a.localName
if (name in arrayOf("__count", "__size", "__type"))
continue
val value = a.value
nodeBuffer.writeU8(46u)
nodeBuffer.writeString(name)
dataBuffer.writeString(value)
}*/
val attributes = e.iterator().asSequence()
.filter { it.localName !in listOf("__type", "__count", "__size") }
.sortedBy { it.localName }
for (a in attributes) {
nodeBuffer.writeU8(46u)
nodeBuffer.writeString(a.localName)
dataBuffer.writeString(a.value)
}
for (c in e.childElements) {
nodeRecurse(c, dataBuffer, nodeBuffer)
}
nodeBuffer.writeU8(254u)
}
}

11
src/main/java/com/buttongames/butterfly/xml/kbinxml/Public.kt Normal file
View File

@ -0,0 +1,11 @@
package com.buttongames.butterfly.xml.kbinxml
import nu.xom.Builder
import nu.xom.Document
fun kbinEncode(d: Document) = KbinWriter(d).getKbin()
fun kbinEncode(s: String) = kbinEncode(Builder().build(s, null))
fun kbinDecode(b: ByteArray) = KbinReader(b).getXml()
fun kbinDecodeToString(b: ByteArray) = kbinDecode(b).prettyString()

46
src/main/java/com/buttongames/butterfly/xml/kbinxml/Sixbit.kt Normal file
View File

@ -0,0 +1,46 @@
package com.buttongames.butterfly.xml.kbinxml
import kotlin.experimental.or
import kotlin.math.roundToInt
internal class Sixbit {
companion object {
private const val characters = "0123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
private val charToByte: Map<Char, Byte> = mutableMapOf()
init {
charToByte as MutableMap<Char, Byte>
for (i in 0 until characters.length) {
charToByte[characters[i]] = i.toByte()
}
}
private fun pack(bytes: ByteArray): ByteArray {
val output = ByteArray(Math.ceil(bytes.size * 6.0 / 8).roundToInt())
for (i in 0 until bytes.size * 6) {
output[i / 8] = output[i / 8] or
((bytes[i / 6].toInt() shr (5 - (i % 6)) and 1)
shl (7 - (i % 8))).toByte()
}
return output
}
fun decode(input: ByteArray, length: Int): String {
val charBytes = ByteArray(length)
for (i in 0 until length * 6) {
charBytes[i / 6] = charBytes[i / 6] or
(((input[i / 8].toInt() shr (7 - (i % 8))) and 1)
shl (5 - (i % 6))).toByte()
}
return charBytes.map { characters[it.toInt()] }.joinToString("")
}
fun encode(input: String): ByteArray {
val bytes = ByteArray(input.length)
for (i in 0 until input.length) {
bytes[i] = charToByte[input[i]]!!
}
return pack(bytes)
}
}
}

65
src/main/java/com/buttongames/butterfly/xml/kbinxml/Util.kt Normal file
View File

@ -0,0 +1,65 @@
package com.buttongames.butterfly.xml.kbinxml
import nu.xom.Attribute
import nu.xom.Element
import java.nio.charset.Charset
fun byteArrayOfInts(vararg ints: Int) = ByteArray(ints.size) { pos -> ints[pos].toByte() }
internal fun ByteArray.unsigned(): UByteArray {
val length = this.size
var result = UByteArray(length) { 0x00u }
for (i in 0 until length) {
result[i] = this[i].toUByte()
}
return result
}
class KbinException internal constructor(override var message: String) : Exception(message)
internal fun Element.addAttribute(key: String, value: String) {
this.addAttribute(Attribute(key, value))
}
internal fun MutableList<Byte>.setOrAdd(pos: Int, byte: Byte) {
if (pos == this.size) {
this.add(byte)
} else {
while (pos > this.size - 1) {
this.add(0)
}
this[pos] = byte
}
}
internal fun MutableList<Byte>.setOrAddAll(pos: Int, bytes: ByteArray) {
var posMut = pos
for (b in bytes) {
this.setOrAdd(posMut, b)
posMut++
}
}
internal fun ByteArray.toString(encoding: String) = this.toString(Charset.forName(encoding))
internal fun String.splitAndJoin(count: Int): Array<String> {
val input = this.split(" ")
val output = Array(input.size / count) { "" }
for (i in 0 until output.size) {
output[i] = input.slice(i * count until (i + 1) * count).joinToString(" ")
}
return output
}
fun measureMs(times: Int = 1, function: () -> Unit) {
for (i in 0 until times) {
val startTime = System.nanoTime()
function()
val endTime = System.nanoTime()
val duration = endTime - startTime //divide by 1000000 to get milliseconds.
println("Took ${duration / 1000000.0} ms")
}
}
internal inline fun Byte.posInt() = this.toUByte().toInt()

78
src/main/java/com/buttongames/butterfly/xml/kbinxml/XmlUtil.kt Normal file
View File

@ -0,0 +1,78 @@
package com.buttongames.butterfly.xml.kbinxml
import nu.xom.*
import java.io.ByteArrayOutputStream
import java.io.File
fun deepCompare(a: Document, b: Document): Boolean {
return deepCompare(a.copy().rootElement, b.copy().rootElement)
}
private fun deepCompare(a: Element, b: Element): Boolean {
a.sortAttributes(); b.sortAttributes()
val nameA = a.localName
val nameB = b.localName
check(nameA == nameB) { "Element names are different: $nameA != $nameB" }
val attrItA = a.iterator()
val attrItB = b.iterator()
while (attrItA.hasNext()) {
val attrA = attrItA.next()
val attrB = attrItB.next()
check(attrA.localName == attrB.localName) { "Attribute names are different: ${attrA.localName} != ${attrB.localName}" }
if (attrB.value == "bib")
println("break")
check(attrA.value == attrB.value) { "Attribute values are different for \"${attrA.localName}\": ${attrA.value} != ${attrB.value}" }
}
if (a.text != "") {
check(a.text == b.text) { "Text inside of tags does not match: ${a.value} != ${b.value}" }
}
val elemItA = a.childElements.iterator()
val elemItB = b.childElements.iterator()
while (elemItA.hasNext()) {
deepCompare(elemItA.next(), elemItB.next())
}
return true
}
fun Document.prettyString(): String {
val o = ByteArrayOutputStream()
val serializer = Serializer(o, "UTF-8")
serializer.setIndent(4)
serializer.write(this)
return o.toString("UTF-8")
}
var Element.text: String
get() {
if (this.childCount == 1) {
val e = this.getChild(0)
if (e is Text) {
return e.value
}
}
return ""
}
set(value: String) {
this.removeChildren()
this.appendChild(value)
}
internal fun Element.sortAttributesRec(): Element {
this.sortAttributes()
for (e in this.childElements) {
sortAttributesRecE(e)
}
return this
}
private fun sortAttributesRecE(e: Element) {
e.sortAttributes()
for (b in e.childElements) {
sortAttributesRecE(b)
}
}
fun String.toXml() = Builder().build(this, null)!!
fun File.toXml() = Builder().build(this)