diff --git a/.gitignore b/.gitignore index 9787819..c4d4e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ .mtj.tmp/ # Package Files # -*.jar *.war *.nar *.ear @@ -33,6 +32,7 @@ gradle-app.setting .gradletasknamecache /bin/ .DS_Store +*.iml # vscode stuff .project diff --git a/build.gradle b/build.gradle index 45c95c7..5a2a728 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,7 @@ group 'com.buttongames' version '1.0-SNAPSHOT' apply plugin: 'java' +apply plugin: 'kotlin' apply plugin: 'application' sourceCompatibility = 1.8 @@ -13,6 +14,8 @@ repositories { } dependencies { + implementation fileTree(dir: 'lib', include: ['*.jar']) + testCompile group: 'junit', name: 'junit', version: '4.12' // Spark, core HTTP server provider @@ -37,6 +40,9 @@ dependencies { compile group: 'org.hibernate', name: 'hibernate-java8', version: '5.4.0.Final' compile group: 'org.springframework', name: 'spring-orm', version: '5.1.3.RELEASE' compile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.25.2' + + // Kotlin, for kbinxml support + compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" } jar { @@ -51,4 +57,26 @@ jar { it.isDirectory() ? it : zipTree(it) } } +} + +buildscript { + ext.kotlin_version = '1.3.11' + repositories { + mavenCentral() + } + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} + +compileKotlin { + kotlinOptions { + jvmTarget = "1.8" + } +} + +compileTestKotlin { + kotlinOptions { + jvmTarget = "1.8" + } } \ No newline at end of file diff --git a/lib/xom-1.3.0-SNAPSHOT.jar b/lib/xom-1.3.0-SNAPSHOT.jar new file mode 100644 index 0000000..bd98757 Binary files /dev/null and b/lib/xom-1.3.0-SNAPSHOT.jar differ diff --git a/src/main/java/com/buttongames/butterfly/http/ButterflyHttpServer.java b/src/main/java/com/buttongames/butterfly/http/ButterflyHttpServer.java index ef7370e..2e14cf1 100644 --- a/src/main/java/com/buttongames/butterfly/http/ButterflyHttpServer.java +++ b/src/main/java/com/buttongames/butterfly/http/ButterflyHttpServer.java @@ -27,8 +27,8 @@ import com.buttongames.butterfly.http.handlers.impl.TaxRequestHandler; import com.buttongames.butterfly.model.ButterflyUser; import com.buttongames.butterfly.model.Machine; import com.buttongames.butterfly.util.PropertyNames; -import com.buttongames.butterfly.xml.BinaryXmlUtils; import com.buttongames.butterfly.xml.XmlUtils; +import com.buttongames.butterfly.xml.kbinxml.PublicKt; import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -299,13 +299,15 @@ public class ButterflyHttpServer { } // convert the body to plaintext XML if it's binary XML - if (BinaryXmlUtils.isBinaryXML(reqBody)) { - reqBody = BinaryXmlUtils.binaryToXml(reqBody); + Element rootNode = null; + + if (XmlUtils.isBinaryXML(reqBody)) { + rootNode = XmlUtils.stringToXmlFile(PublicKt.kbinDecodeToString(reqBody)); + } else { + rootNode = XmlUtils.byteArrayToXmlFile(reqBody); } // read the request body into an XML document - Element rootNode = XmlUtils.byteArrayToXmlFile(reqBody); - if (rootNode == null || !rootNode.getNodeName().equals("call")) { throw new InvalidRequestException(); diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/BaseRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/BaseRequestHandler.java index 0894c8b..a333b23 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/BaseRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/BaseRequestHandler.java @@ -1,8 +1,8 @@ package com.buttongames.butterfly.http.handlers; -import com.buttongames.butterfly.compression.Lz77; import com.buttongames.butterfly.encryption.Rc4; -import com.buttongames.butterfly.xml.BinaryXmlUtils; +import com.buttongames.butterfly.xml.XmlUtils; +import com.buttongames.butterfly.xml.kbinxml.PublicKt; import com.google.common.net.MediaType; import com.jamesmurty.utils.BaseXMLBuilder; import org.apache.logging.log4j.LogManager; @@ -27,7 +27,6 @@ import java.nio.file.Paths; import static com.buttongames.butterfly.util.Constants.COMPRESSION_HEADER; import static com.buttongames.butterfly.util.Constants.CRYPT_KEY_HEADER; -import static com.buttongames.butterfly.util.Constants.LZ77_COMPRESSION; /** * Base request handler that the others inherit from. @@ -106,8 +105,8 @@ public abstract class BaseRequestHandler { response.header("Connection", "keep-alive"); // convert them to binary XML - if (!BinaryXmlUtils.isBinaryXML(respBytes)) { - respBytes = BinaryXmlUtils.xmlToBinary(respBytes); + if (!XmlUtils.isBinaryXML(respBytes)) { + respBytes = PublicKt.kbinEncode(new String(respBytes)); } // TODO: FIX THIS SHIT diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/CardManageRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/CardManageRequestHandler.java index bec886e..42eb32a 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/CardManageRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/CardManageRequestHandler.java @@ -11,7 +11,7 @@ import com.buttongames.butterfly.model.CardType; import com.buttongames.butterfly.util.CardIdUtils; import com.buttongames.butterfly.util.StringUtils; import com.buttongames.butterfly.xml.XmlUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/EventLogRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/EventLogRequestHandler.java index 904a73a..08b8e80 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/EventLogRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/EventLogRequestHandler.java @@ -5,7 +5,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.model.ddr16.GameplayEventLog; import com.buttongames.butterfly.util.TimeUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import com.buttongames.butterfly.xml.XmlUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/FacilityRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/FacilityRequestHandler.java index 416e55d..57f3c1f 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/FacilityRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/FacilityRequestHandler.java @@ -6,7 +6,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.model.ddr16.Shop; import com.buttongames.butterfly.util.StringUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/MessageRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/MessageRequestHandler.java index 5372e89..84faf5e 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/MessageRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/MessageRequestHandler.java @@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl; import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.util.PropertyNames; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PackageRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PackageRequestHandler.java index 43791cc..046f1fa 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PackageRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PackageRequestHandler.java @@ -2,7 +2,7 @@ package com.buttongames.butterfly.http.handlers.impl; import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbEventRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbEventRequestHandler.java index 995ae64..8ef12a3 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbEventRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbEventRequestHandler.java @@ -5,7 +5,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.model.ddr16.PcbEventLog; import com.buttongames.butterfly.util.TimeUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import com.buttongames.butterfly.xml.XmlUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbTrackerRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbTrackerRequestHandler.java index 6eeb82c..2ec6dc2 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbTrackerRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PcbTrackerRequestHandler.java @@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl; import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.util.PropertyNames; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PlayerDataRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PlayerDataRequestHandler.java index 8a6e617..e4920a5 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/PlayerDataRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/PlayerDataRequestHandler.java @@ -25,7 +25,7 @@ import com.buttongames.butterfly.model.ddr16.options.SpeedOption; import com.buttongames.butterfly.model.ddr16.options.StepZoneOption; import com.buttongames.butterfly.model.ddr16.options.TurnOption; import com.buttongames.butterfly.util.StringUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import com.buttongames.butterfly.xml.XmlUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/ServicesRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/ServicesRequestHandler.java index a68a4db..3dc7590 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/ServicesRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/ServicesRequestHandler.java @@ -3,7 +3,7 @@ package com.buttongames.butterfly.http.handlers.impl; import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.util.PropertyNames; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import com.google.common.collect.ImmutableMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/SystemRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/SystemRequestHandler.java index 1a74851..7df974b 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/SystemRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/SystemRequestHandler.java @@ -4,7 +4,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.util.CardIdUtils; import com.buttongames.butterfly.xml.XmlUtils; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/buttongames/butterfly/http/handlers/impl/TaxRequestHandler.java b/src/main/java/com/buttongames/butterfly/http/handlers/impl/TaxRequestHandler.java index 561e688..857ca2d 100644 --- a/src/main/java/com/buttongames/butterfly/http/handlers/impl/TaxRequestHandler.java +++ b/src/main/java/com/buttongames/butterfly/http/handlers/impl/TaxRequestHandler.java @@ -6,7 +6,7 @@ import com.buttongames.butterfly.http.exception.InvalidRequestMethodException; import com.buttongames.butterfly.http.handlers.BaseRequestHandler; import com.buttongames.butterfly.model.Machine; import com.buttongames.butterfly.model.UserPhases; -import com.buttongames.butterfly.xml.builder.KXmlBuilder; +import com.buttongames.butterfly.xml.kbinxml.KXmlBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/main/java/com/buttongames/butterfly/xml/BinaryXmlUtils.java b/src/main/java/com/buttongames/butterfly/xml/BinaryXmlUtils.java deleted file mode 100644 index 48e9efc..0000000 --- a/src/main/java/com/buttongames/butterfly/xml/BinaryXmlUtils.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.buttongames.butterfly.xml; - -import com.google.common.io.ByteStreams; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - -/** - * Class to help translating between binary and plaintext XML. Right now it's a very dumb - * class and just uses mon's Python implementation until I can make a native Java one. This does - * depend on you having done a pip install kbinxml to make kbinxml - * a valid command. - * See: https://github.com/mon/kbinxml - * @author skogaby (skogabyskogaby@gmail.com) - */ -public class BinaryXmlUtils { - - /** - * First bytes of a plaintext XML response, so we know if an array needs to be converted - * to/from binary XML. - */ - private static final byte[] XML_PREFIX = " + + init { + val m1 = mutableMapOf() + for (i in 1 until encodings.size) { + m1[encodings[i]] = i + } + encodingsReverse = m1 + } + } +} + +internal enum class ControlTypes { + NodeStart, + Attribute, + NodeEnd, + FileEnd +} + +internal val ControlTypeMap = mapOf( + 1 to ControlTypes.NodeStart, + 46 to ControlTypes.Attribute, + 190 to ControlTypes.NodeEnd, + 191 to ControlTypes.FileEnd + //254 to ControlTypes.NodeEnd, + //255 to ControlTypes.FileEnd +) + +internal class KbinConverter(val fromString: (String) -> ByteArray, val toString: (ByteArray) -> String) + +internal class KbinType(var names: List, val size: Int, private val handler: KbinConverter, val count: Int = 1) { + + constructor(name: String, size: Int, handler: KbinConverter, count: Int = 1) : this(mutableListOf(name), size, handler, count) + + val name: String + get() = names[0] + + fun alias(alias: String): KbinType { + (names as MutableList) += alias + return this + } + + fun rename(name: String): KbinType { + val mutable = (names as MutableList) + mutable.clear() + mutable.add(name) + return this + } + + fun fromString(string: String): ByteArray = if (string.isEmpty()) byteArrayOf() else handler.fromString(string) + fun toString(bytes: ByteArray): String = if (bytes.isEmpty()) "" else handler.toString(bytes) +} + +internal operator fun Int.times(t: KbinType): KbinType { + val newNames = t.names.map { this.toString() + it } + val newSize = t.size * this + fun newToString(input: ByteArray) = + input.asIterable().chunked(t.size).joinToString(" ") { t.toString(it.toByteArray()) } + + fun newFromString(input: String) = + input.split(" ").flatMap { t.fromString(it).asIterable() }.toByteArray() + + return KbinType(newNames, newSize, KbinConverter(::newFromString, ::newToString), this * t.count) +} + +internal class Types { + companion object { + val s8 = KbinType("s8", 1, Converters.s8) + val u8 = KbinType("u8", 1, Converters.u8) + val s16 = KbinType("s16", 2, Converters.s16) + val u16 = KbinType("u16", 2, Converters.u16) + val s32 = KbinType("s32", 4, Converters.s32) + val u32 = KbinType("u32", 4, Converters.u32) + val s64 = KbinType("s64", 8, Converters.s64) + val u64 = KbinType("u64", 8, Converters.u64) + + val bin = KbinType(listOf("bin", "binary"), 1, Converters.stub) + val strStub = KbinType(listOf("str", "string"), 1, Converters.stub) + + val ip4 = KbinType("ip4", 4, Converters.ip4) + val time = KbinType("time", 4, Converters.u32) + val float = KbinType(listOf("float", "f"), 4, Converters.float) + val double = KbinType(listOf("double", "d"), 8, Converters.double) + val bool = KbinType(listOf("bool", "b"), 1, Converters.bool) + } +} + +internal val kbinTypeMap = with(Types) { + mapOf( + 2 to s8, + 3 to u8, + 4 to s16, + 5 to u16, + 6 to s32, + 7 to u32, + 8 to s64, + 9 to u64, + 10 to bin, + 11 to strStub, + 12 to ip4, + 13 to time, + 14 to float, + 15 to double, + 16 to 2 * s8, + 17 to 2 * u8, + 18 to 2 * s16, + 19 to 2 * u16, + 20 to 2 * s32, + 21 to 2 * u32, + 22 to (2 * s64).alias("vs64"), + 23 to (2 * u64).alias("vu64"), + 24 to (2 * float).rename("2f"), + 25 to (2 * double).rename("2d").alias("vd"), + 26 to 3 * s8, + 27 to 3 * u8, + 28 to 3 * s16, + 29 to 3 * u16, + 30 to 3 * s32, + 31 to 3 * u32, + 32 to 3 * s64, + 33 to 3 * u64, + 34 to (3 * float).rename("3f"), + 35 to (3 * double).rename("3d"), + 36 to 4 * s8, + 37 to 4 * u8, + 38 to 4 * s16, + 39 to 4 * u16, + 40 to (4 * s32).alias("vs32"), + 41 to (4 * u32).alias("vu32"), + 42 to 4 * s64, + 43 to 4 * u64, + 44 to (4 * float).rename("4f").alias("vf"), + 45 to (4 * double).rename("4d"), + 48 to (16 * s8).rename("vs8"), + 49 to (16 * u8).rename("vu8"), + 50 to (8 * s16).rename("vs16"), + 51 to (8 * u16).rename("vu16"), + 52 to bool, + 53 to (2 * bool).rename("2b"), + 54 to (3 * bool).rename("3b"), + 55 to (4 * bool).rename("4b"), + 56 to (16 * bool).rename("vb") + ) +} + +internal var reverseKbinTypeMap: Map = kbinTypeMap.entries.associateBy({ it.value.name }) { it.key } \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/builder/KXmlBuilder.java b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KXmlBuilder.java similarity index 99% rename from src/main/java/com/buttongames/butterfly/xml/builder/KXmlBuilder.java rename to src/main/java/com/buttongames/butterfly/xml/kbinxml/KXmlBuilder.java index d229555..ce9a69c 100644 --- a/src/main/java/com/buttongames/butterfly/xml/builder/KXmlBuilder.java +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KXmlBuilder.java @@ -1,4 +1,4 @@ -package com.buttongames.butterfly.xml.builder; +package com.buttongames.butterfly.xml.kbinxml; /** * CONTEXT: I (skogaby) wanted to extend XMLBuilder2 from java-xmlbuilder to provide convenience diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinConverters.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinConverters.kt new file mode 100644 index 0000000..dabe0b6 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinConverters.kt @@ -0,0 +1,23 @@ +package com.buttongames.butterfly.xml.kbinxml + +internal class Converters { + companion object { + val s8 = KbinConverter({ it.toByteA() }) { it.toByte().toString() } + val u8 = KbinConverter({ it.toUByteA() }) { it.toUByte().toString() } + val s16 = KbinConverter({ it.toShortBytes() }) { it.toShort().toString() } + val u16 = KbinConverter({ it.toUShortBytes() }) { it.toUShort().toString() } + val s32 = KbinConverter({ it.toIntBytes() }) { it.toInt().toString() } + val u32 = KbinConverter({ it.toUIntBytes() }) { it.toUInt().toString() } + val s64 = KbinConverter({ it.toLongBytes() }) { it.toLong().toString() } + val u64 = KbinConverter({ it.toULongBytes() }) { it.toULong().toString() } + + // val bin = KbinConverter(ByteConv.E::stringToBin, ByteConv.E::binToString) + val stub = KbinConverter({ "STUB".toByteArray() }, { "STUB" }) // scary + + val ip4 = KbinConverter(ByteConv.E::stringToIp, ByteConv.E::ipToString) + val float = KbinConverter(ByteConv.E::stringToFloat, ByteConv.E::floatToString) + val double = KbinConverter(ByteConv.E::stringToDouble, ByteConv.E::doubleToString) + + val bool = KbinConverter(ByteConv.E::stringToBool, ByteConv.E::boolToString) + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinDataBuffer.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinDataBuffer.kt new file mode 100644 index 0000000..4bc081e --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinDataBuffer.kt @@ -0,0 +1,161 @@ +package com.buttongames.butterfly.xml.kbinxml + +import java.nio.charset.Charset + +internal class KbinDataBuffer(bytes: ByteArray, val encoding: Charset) { + constructor(encoding: Charset) : this(byteArrayOf(), encoding) + + private val data = bytes.toMutableList() + + val size: Int + get() = data.size + + private var pos8 = 0 + private var pos16 = 0 + private var pos32 = 0 + + fun readBytes(num: Int): ByteArray { + val result: ByteArray + + val debug: String + + when { + num == 0 -> return byteArrayOf() + num == 1 -> { + result = byteArrayOf(data[pos8]) + debug = "$pos8" + } + num == 2 -> { + result = data.slice(pos16 until pos16 + 2).toByteArray() + debug = "$pos16 - ${pos16 + 2 - 1}" + } + num >= 3 -> { + result = data.slice(pos32 until pos32 + num).toByteArray() + debug = "$pos32 - ${pos32 + num - 1}" + } + else -> throw KbinException("Invalid read of $num bytes") + } + realign(num) + // println("Read bytes $debug") + return result + } + + private fun realign(bytesRead: Int) { + fun pos8Follows() = pos8 % 4 == 0 + fun pos16Follows() = pos16 % 4 == 0 + + if (bytesRead == 1) { + if (pos8Follows()) { + pos32 += 4 + } + pos8++ + } + if (bytesRead == 2) { + if (pos16Follows()) { + pos32 += 4 + } + pos16 += 2 + } + if (bytesRead >= 3) { + var newNum = bytesRead + if (newNum % 4 != 0) newNum += 4 - (newNum % 4) + pos32 += newNum + } + if (pos8Follows()) { + pos8 = pos32 + } + if (pos16Follows()) { + pos16 = pos32 + } + //println("Index 4: $pos32") + //println("Index 2: $pos16") + //println("Index 1: $pos8") + } + + fun realign4Byte(num: Int) { + realign(num + (if (num % 4 != 0) { + 4 - (num % 4) + } else 0)) + } + + fun reset() { + pos8 = 0; pos16 = 0; pos32 = 0 + } + + fun readU8(): UByte { + val result = readBytes(1)[0].toUByte() + return result + } + + fun readU16(): UShort { + val result = readBytes(2) + return result.toUShort() + } + + fun readU32(): UInt { + val result = readBytes(4) + return result.toUInt() + } + + fun readFrom4Byte(num: Int): ByteArray { + if (num == 0) return byteArrayOf() + val read = data.slice(pos32 until pos32 + num) + realign4Byte(num) + return read.toByteArray() + } + + fun readString(length: Int): String { + var readBytes = readFrom4Byte(length) + if (readBytes.last() == 0x00.toByte()) { // null bytes are scary + readBytes = readBytes.sliceArray(0 until readBytes.lastIndex) + } + return readBytes.toString(encoding) + } + + fun writeBytes(bytes: ByteArray) { + val length = bytes.size + when { + length == 0 -> return + length == 1 -> { + data.setOrAddAll(pos8, bytes) + } + length == 2 -> { + data.setOrAddAll(pos16, bytes) + } + length >= 3 -> { + data.setOrAddAll(pos32, bytes) + } + else -> { + throw KbinException("Invalid write of $length bytes") + } + } + realign(length) + } + + fun writeU8(value: UByte) { + writeBytes(byteArrayOf(value.toByte())) + } + + fun writeU32(value: UInt) { + writeBytes(value.toInt().toByteArray()) + } + + fun writeTo4Byte(bytes: ByteArray) { + data.setOrAddAll(pos32, bytes) + realign4Byte(bytes.size) + } + + fun writeString(string: String) { + var bytes = string.toByteArray(encoding) + 0 // null byte + writeU32(bytes.size.toUInt()) + writeTo4Byte(bytes) + } + + fun getContent(): ByteArray { + return data.toByteArray() + } + + fun pad() { + while (data.size % 4 != 0) data.add(0) + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinNodeBuffer.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinNodeBuffer.kt new file mode 100644 index 0000000..1ba9f7c --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinNodeBuffer.kt @@ -0,0 +1,67 @@ +package com.buttongames.butterfly.xml.kbinxml + +import java.nio.charset.Charset +import kotlin.math.roundToInt + +internal class KbinNodeBuffer(bytes: ByteArray, val compressed: Boolean, val encoding: Charset) { + constructor (compressed: Boolean, encoding: Charset) : this(byteArrayOf(), compressed, encoding) + + private val data = bytes.toMutableList() + private var index = 0 + + val size: Int + get() = data.size + + fun readU8() = readBytes(1)[0].toUByte() + + fun readBytes(num: Int): ByteArray { + val result = data.slice(index until index + num).toByteArray() + index += num + return result + } + + fun reset() { + index = 0 + } + + fun readString(): String { + val length = readU8().toInt() + if (compressed) { + val toRead = Math.ceil(length * 6 / 8.0).roundToInt() + val nameBytes = readBytes(toRead) + return Sixbit.decode(nameBytes, length) + } else { + val readBytes = readBytes((length and 64.inv()) + 1) + return readBytes.toString(encoding) + } + } + + fun writeBytes(bytes: ByteArray) { + data.setOrAddAll(index, bytes) + index += bytes.size + } + + fun writeU8(byte: UByte) { + writeBytes(byteArrayOf(byte.toByte())) + } + + fun writeString(string: String) { + val bytes: ByteArray + if (compressed) { + bytes = Sixbit.encode(string) + writeU8(string.length.toUByte()) + } else { + bytes = string.toByteArray(encoding) + writeU8(((bytes.size - 1) or (1 shl 6)).toUByte()) + } + writeBytes(bytes) + } + + fun getContent(): ByteArray { + return data.toByteArray() + } + + fun pad() { + while (data.size % 4 != 0) data.add(0) + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinReader.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinReader.kt new file mode 100644 index 0000000..f90a5b6 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinReader.kt @@ -0,0 +1,133 @@ +package com.buttongames.butterfly.xml.kbinxml + +import nu.xom.Document +import nu.xom.Element +import com.buttongames.butterfly.xml.kbinxml.ControlTypes.* +import java.nio.charset.Charset +import java.util.* +import kotlin.experimental.inv + +internal class KbinReader(data: ByteArray) { + + init { + if (data[0] != 0xA0.toByte()) { + throw KbinException("First byte must be 0xA0") + } + } + + val compressed = when (data[1].toInt()) { + 0x42 -> true + 0x45 -> false + else -> throw KbinException("Second byte of data must be 0x42 or 0x45") + } + + val encoding = when ( + val tmp = data[2].posInt() shr 5) { + in 0..5 -> Charset.forName(Constants.encodings[tmp]) + else -> throw KbinException("Third byte does not match any encoding") + } + + init { + if (data[2] != data[3].inv()) throw KbinException("Fourth byte must be inverse of third") + } + + private val nodeBuffer: KbinNodeBuffer + private val dataBuffer: KbinDataBuffer + + init { + val nodeLength = data.slice(4 until 8).toByteArray().toUInt().toInt() + nodeBuffer = KbinNodeBuffer(data.slice(8 until (8 + nodeLength)).toByteArray(), compressed, encoding) + val dataStart = 12 + nodeLength + val dataLength = data.slice(dataStart - 4 until dataStart).toByteArray().toUInt().toInt() + dataBuffer = KbinDataBuffer(data.slice(dataStart until (dataStart + dataLength)).toByteArray(), encoding) + } + + fun getXml(): Document { + nodeBuffer.reset(); dataBuffer.reset() + var currentNode: Element? = null + val nodeStack = ArrayDeque() + var end = false + + while (!end) { + val current = nodeBuffer.readU8().toInt() + val actual = current and (1 shl 6).inv() + val controlType = ControlTypeMap[actual] + val valueType = kbinTypeMap[actual] + + if (controlType != null) + when (controlType) { + NodeStart -> { + val name = nodeBuffer.readString() + + val newNode = Element(name) + if (currentNode != null) { + currentNode.appendChild(newNode) + nodeStack.push(currentNode) + } + currentNode = newNode + } + NodeEnd -> { + if (nodeStack.size > 0) { + //println("Popping Node ${nodeStack.peek().localName}") + //currentNode?.sortAttributes() + currentNode = nodeStack.pop() + } + } + Attribute -> { + val name = nodeBuffer.readString() + + val valueLength = dataBuffer.readU32().toInt() + val value = dataBuffer.readString(valueLength) + + //println("Got attribute $name with value $value") + + currentNode!!.addAttribute(name, value) + } + FileEnd -> { + if (nodeStack.size == 0) { + end = true + } else throw KbinException("Byte indicates end of file, but parsing is not done") + } + } + else if (valueType != null) { + val valueName = valueType.name + val isArray = (((current shr 6) and 1) == 1) or (valueName in listOf("bin", "str")) + val nodeName = nodeBuffer.readString() + val arraySize = if (isArray) dataBuffer.readU32().toInt() else valueType.size + + nodeStack.push(currentNode) + val newNode = Element(nodeName) + newNode.addAttribute("__type", valueName) + currentNode!!.appendChild(newNode) + currentNode = newNode + + val numElements = arraySize / valueType.size + when (valueName) { + "bin" -> { + currentNode.addAttribute("__size", arraySize.toString()) + val bytes = dataBuffer.readFrom4Byte(arraySize) + currentNode.text = ByteConv.binToString(bytes) + } + "str" -> { + currentNode.text = dataBuffer.readString(arraySize) + } + else -> { + if (isArray) currentNode.addAttribute("__count", numElements.toString()) + + val byteList = dataBuffer.readBytes(arraySize) + val stringList = mutableListOf() + for (i in 0 until numElements) { + val bytes = byteList.sliceArray(i * valueType.size until (i + 1) * valueType.size) + stringList.add(valueType.toString(bytes)) + } + currentNode.text = stringList.joinToString(separator = " ") + } + + } + } else if (current != 1) { + throw KbinException("Unsupported node type with ID $actual") + } + } + return Document(currentNode) + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinWriter.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinWriter.kt new file mode 100644 index 0000000..7273960 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/KbinWriter.kt @@ -0,0 +1,98 @@ +package com.buttongames.butterfly.xml.kbinxml + +import nu.xom.Document +import nu.xom.Element +import java.io.ByteArrayOutputStream +import java.nio.charset.Charset +import kotlin.experimental.inv + +internal class KbinWriter(val xml: Document, val encoding: String = "SHIFT_JIS", val compressed: Boolean = true) { + private val header = ByteArray(4) + + val charset = Charset.forName(encoding) + + fun getKbin(): ByteArray { + val dataBuffer = KbinDataBuffer(charset) + val nodeBuffer = KbinNodeBuffer(compressed, charset) + + header[0] = 0xa0u.toByte() + header[1] = (if (compressed) 0x42u else 0x45u).toByte() + header[2] = (Constants.encodingsReverse[encoding]!! shl 5).toByte() + header[3] = header[2].inv() + + nodeRecurse(xml.rootElement, dataBuffer, nodeBuffer) + + nodeBuffer.writeU8(255u) + nodeBuffer.pad() + dataBuffer.pad() + val output = ByteArrayOutputStream() + output.write(header) + output.write(nodeBuffer.size.toByteArray()) + output.write(nodeBuffer.getContent()) + output.write(dataBuffer.size.toByteArray()) + output.write(dataBuffer.getContent()) + return output.toByteArray() + + } + + private fun nodeRecurse(e: Element, dataBuffer: KbinDataBuffer, nodeBuffer: KbinNodeBuffer) { + val typeName = e.getAttribute("__type")?.value + val typeId = reverseKbinTypeMap[typeName] + if (typeName != null && typeId == null) throw KbinException("Type $typeName is not supported") + val type = kbinTypeMap[typeId] + val count = (e.getAttribute("__count")?.value ?: e.getAttribute("__size")?.value)?.toInt() + + val isArray = (count != null) && type?.name !in listOf("bin", "str") + + if (typeId == null) { + nodeBuffer.writeU8(1u) + } else { + var toWrite = typeId.toUByte() + if (isArray) { + toWrite = toWrite or ((1 shl 6).toUByte()) + } + nodeBuffer.writeU8(toWrite) + } + nodeBuffer.writeString(e.localName) + if (type != null) { + if (type.name == "bin") { + if (count != null) { + dataBuffer.writeU32(count.toUInt()) + } + val toWrite = ByteConv.stringToBin(e.text) + dataBuffer.writeTo4Byte(toWrite) + } else if (type.name == "str") { + dataBuffer.writeString(e.text) + } else { + if (count != null) { + dataBuffer.writeU32((count * type.size).toUInt()) + } + val split = e.text.splitAndJoin(type.count) + val toWrite = split.flatMap { type.fromString(it).asIterable() }.toByteArray() + dataBuffer.writeBytes(toWrite) + } + } + /*for (a in e) { + val name = a.localName + if (name in arrayOf("__count", "__size", "__type")) + continue + val value = a.value + nodeBuffer.writeU8(46u) + nodeBuffer.writeString(name) + dataBuffer.writeString(value) + }*/ + val attributes = e.iterator().asSequence() + .filter { it.localName !in listOf("__type", "__count", "__size") } + .sortedBy { it.localName } + + for (a in attributes) { + nodeBuffer.writeU8(46u) + nodeBuffer.writeString(a.localName) + dataBuffer.writeString(a.value) + } + for (c in e.childElements) { + nodeRecurse(c, dataBuffer, nodeBuffer) + } + nodeBuffer.writeU8(254u) + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/Public.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Public.kt new file mode 100644 index 0000000..da304a4 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Public.kt @@ -0,0 +1,11 @@ +package com.buttongames.butterfly.xml.kbinxml + +import nu.xom.Builder +import nu.xom.Document + +fun kbinEncode(d: Document) = KbinWriter(d).getKbin() +fun kbinEncode(s: String) = kbinEncode(Builder().build(s, null)) + +fun kbinDecode(b: ByteArray) = KbinReader(b).getXml() + +fun kbinDecodeToString(b: ByteArray) = kbinDecode(b).prettyString() diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/Sixbit.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Sixbit.kt new file mode 100644 index 0000000..2310ce3 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Sixbit.kt @@ -0,0 +1,46 @@ +package com.buttongames.butterfly.xml.kbinxml + +import kotlin.experimental.or +import kotlin.math.roundToInt + +internal class Sixbit { + companion object { + private const val characters = "0123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" + private val charToByte: Map = mutableMapOf() + + init { + charToByte as MutableMap + for (i in 0 until characters.length) { + charToByte[characters[i]] = i.toByte() + } + } + + private fun pack(bytes: ByteArray): ByteArray { + val output = ByteArray(Math.ceil(bytes.size * 6.0 / 8).roundToInt()) + for (i in 0 until bytes.size * 6) { + output[i / 8] = output[i / 8] or + ((bytes[i / 6].toInt() shr (5 - (i % 6)) and 1) + shl (7 - (i % 8))).toByte() + } + return output + } + + fun decode(input: ByteArray, length: Int): String { + val charBytes = ByteArray(length) + for (i in 0 until length * 6) { + charBytes[i / 6] = charBytes[i / 6] or + (((input[i / 8].toInt() shr (7 - (i % 8))) and 1) + shl (5 - (i % 6))).toByte() + } + return charBytes.map { characters[it.toInt()] }.joinToString("") + } + + fun encode(input: String): ByteArray { + val bytes = ByteArray(input.length) + for (i in 0 until input.length) { + bytes[i] = charToByte[input[i]]!! + } + return pack(bytes) + } + } +} \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/Util.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Util.kt new file mode 100644 index 0000000..0e0ad6e --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/Util.kt @@ -0,0 +1,65 @@ +package com.buttongames.butterfly.xml.kbinxml + +import nu.xom.Attribute +import nu.xom.Element +import java.nio.charset.Charset + +fun byteArrayOfInts(vararg ints: Int) = ByteArray(ints.size) { pos -> ints[pos].toByte() } + +internal fun ByteArray.unsigned(): UByteArray { + val length = this.size + var result = UByteArray(length) { 0x00u } + for (i in 0 until length) { + result[i] = this[i].toUByte() + } + return result +} + +class KbinException internal constructor(override var message: String) : Exception(message) + +internal fun Element.addAttribute(key: String, value: String) { + this.addAttribute(Attribute(key, value)) +} + +internal fun MutableList.setOrAdd(pos: Int, byte: Byte) { + if (pos == this.size) { + this.add(byte) + } else { + while (pos > this.size - 1) { + this.add(0) + } + this[pos] = byte + } +} + +internal fun MutableList.setOrAddAll(pos: Int, bytes: ByteArray) { + var posMut = pos + for (b in bytes) { + this.setOrAdd(posMut, b) + posMut++ + } +} + +internal fun ByteArray.toString(encoding: String) = this.toString(Charset.forName(encoding)) + +internal fun String.splitAndJoin(count: Int): Array { + val input = this.split(" ") + val output = Array(input.size / count) { "" } + for (i in 0 until output.size) { + output[i] = input.slice(i * count until (i + 1) * count).joinToString(" ") + } + return output +} + +fun measureMs(times: Int = 1, function: () -> Unit) { + for (i in 0 until times) { + val startTime = System.nanoTime() + function() + val endTime = System.nanoTime() + + val duration = endTime - startTime //divide by 1000000 to get milliseconds. + println("Took ${duration / 1000000.0} ms") + } +} + +internal inline fun Byte.posInt() = this.toUByte().toInt() \ No newline at end of file diff --git a/src/main/java/com/buttongames/butterfly/xml/kbinxml/XmlUtil.kt b/src/main/java/com/buttongames/butterfly/xml/kbinxml/XmlUtil.kt new file mode 100644 index 0000000..93a0e63 --- /dev/null +++ b/src/main/java/com/buttongames/butterfly/xml/kbinxml/XmlUtil.kt @@ -0,0 +1,78 @@ +package com.buttongames.butterfly.xml.kbinxml + +import nu.xom.* +import java.io.ByteArrayOutputStream +import java.io.File + +fun deepCompare(a: Document, b: Document): Boolean { + return deepCompare(a.copy().rootElement, b.copy().rootElement) +} + +private fun deepCompare(a: Element, b: Element): Boolean { + a.sortAttributes(); b.sortAttributes() + val nameA = a.localName + val nameB = b.localName + check(nameA == nameB) { "Element names are different: $nameA != $nameB" } + val attrItA = a.iterator() + val attrItB = b.iterator() + while (attrItA.hasNext()) { + val attrA = attrItA.next() + val attrB = attrItB.next() + check(attrA.localName == attrB.localName) { "Attribute names are different: ${attrA.localName} != ${attrB.localName}" } + if (attrB.value == "bib") + println("break") + check(attrA.value == attrB.value) { "Attribute values are different for \"${attrA.localName}\": ${attrA.value} != ${attrB.value}" } + } + if (a.text != "") { + check(a.text == b.text) { "Text inside of tags does not match: ${a.value} != ${b.value}" } + } + val elemItA = a.childElements.iterator() + val elemItB = b.childElements.iterator() + while (elemItA.hasNext()) { + deepCompare(elemItA.next(), elemItB.next()) + } + return true +} + +fun Document.prettyString(): String { + val o = ByteArrayOutputStream() + val serializer = Serializer(o, "UTF-8") + serializer.setIndent(4) + serializer.write(this) + return o.toString("UTF-8") +} + +var Element.text: String + get() { + if (this.childCount == 1) { + val e = this.getChild(0) + if (e is Text) { + return e.value + } + } + return "" + } + set(value: String) { + this.removeChildren() + this.appendChild(value) + } + + +internal fun Element.sortAttributesRec(): Element { + this.sortAttributes() + for (e in this.childElements) { + sortAttributesRecE(e) + } + return this +} + +private fun sortAttributesRecE(e: Element) { + e.sortAttributes() + for (b in e.childElements) { + sortAttributesRecE(b) + } +} + +fun String.toXml() = Builder().build(this, null)!! + +fun File.toXml() = Builder().build(this) \ No newline at end of file