lib: update to 0.13.1, rename exported functions, include a quick_xml NodeCollection reader

This commit is contained in:
Matt Bilker
2018-10-28 06:31:44 +00:00
parent 0a5d07f592
commit 03170b2cd6
9 changed files with 420 additions and 63 deletions

View File

@@ -1,13 +1,13 @@
[package]
name = "kbinxml"
version = "0.12.1"
version = "0.13.1"
authors = ["Matt Bilker <me@mbilker.us>"]
[dependencies]
byteorder = "1.2.3"
bytes = "0.4.10"
cfg-if = "0.1"
encoding = "0.2"
encoding_rs = { version = "0.8.6", features = ["simd-accel"] }
failure = "0.1.1"
indexmap = "1.0.1"
lazy_static = "1.0.0"

View File

@@ -11,12 +11,11 @@ extern crate quick_xml;
use std::env;
use std::fs::File;
use std::io::{Cursor, Error as IoError, ErrorKind as IoErrorKind, Read, Write, stdout};
use std::str;
use failure::Fail;
use kbinxml::{Options, Printer};
use kbinxml::{NodeCollection, Options, Printer};
use minidom::Element;
use quick_xml::Writer;
use quick_xml::{Reader, Writer};
cfg_if! {
if #[cfg(feature = "serde")] {
@@ -86,6 +85,34 @@ fn display_buf(buf: &[u8]) -> Result<(), IoError> {
Ok(())
}
fn compare_collections(left: &NodeCollection, right: &NodeCollection) -> bool {
if left.base() != right.base() {
eprintln!("left.base() != right.base()");
eprintln!("left.base(): {:#?}", left.base());
eprintln!("right.base(): {:#?}", right.base());
return false;
}
for (left, right) in left.attributes().iter().zip(right.attributes().iter()) {
if left != right {
eprintln!("left attribute != right attribute");
eprintln!("left: {:#?}", left);
eprintln!("right: {:#?}", right);
return false;
}
}
for (left, right) in left.children().iter().zip(right.children().iter()) {
if !compare_collections(left, right) {
return false;
}
}
true
}
fn compare_slice(left: &[u8], right: &[u8]) {
let mut buf = [0; 4];
buf.clone_from_slice(&left[4..8]);
@@ -206,7 +233,7 @@ fn main() -> std::io::Result<()> {
if kbinxml::is_binary_xml(&contents) {
Printer::run(&contents).unwrap();
let (element, encoding_original) = kbinxml::from_binary(&contents).map_err(display_err)?;
let (element, encoding_original) = kbinxml::element_from_binary(&contents).map_err(display_err)?;
let text_original = to_text(&element)?;
display_buf(&text_original)?;
@@ -216,13 +243,19 @@ fn main() -> std::io::Result<()> {
test_serde_node(&contents)?;
} else {
let contents = str::from_utf8(&contents).expect("Unable to interpret file contents as UTF-8");
let element: Element = contents.parse().expect("Unable to construct DOM for input text XML");
let mut reader = Reader::from_reader(contents.as_slice());
let element = Element::from_reader(&mut reader).expect("Unable to construct DOM for input text XML");
let options = Options::default();
let buf = kbinxml::to_binary_with_options(options, &element).map_err(display_err)?;
eprintln!("data: {:02x?}", buf);
Printer::run(&buf).unwrap();
let encoded_collection = Printer::run(&buf).unwrap();
let (collection, _encoding) = kbinxml::from_text_xml(&contents).map_err(display_err)?;
if let Some(encoded_collection) = encoded_collection {
compare_collections(&encoded_collection, &collection);
}
let mut stdout = stdout();
stdout.lock().write_all(&buf)?;

View File

@@ -1,7 +1,11 @@
use error::{KbinError, KbinErrorKind};
use failure::ResultExt;
use encoding::{DecoderTrap, EncoderTrap, Encoding};
use encoding::all::{ASCII, EUC_JP, ISO_8859_1, WINDOWS_31J};
/// The `encoding_rs` crate uses the following to describe their counterparts:
///
/// `SHIFT_JIS` => `WINDOWS_31J`
/// `WINDOWS_1252` => `ISO-8859-1`
use encoding_rs::{Encoding, EUC_JP, SHIFT_JIS, UTF_8, WINDOWS_1252};
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -35,6 +39,18 @@ impl EncodingType {
Ok(val)
}
pub fn from_encoding(encoding: &'static Encoding) -> Result<Self, KbinError> {
let val = match encoding {
e if e == WINDOWS_1252 => EncodingType::ISO_8859_1,
e if e == EUC_JP => EncodingType::EUC_JP,
e if e == SHIFT_JIS => EncodingType::SHIFT_JIS,
e if e == UTF_8 => EncodingType::UTF_8,
_ => return Err(KbinErrorKind::UnknownEncoding.into()),
};
Ok(val)
}
pub fn to_byte(&self) -> u8 {
match *self {
EncodingType::None => 0x00, // 0x00 >> 5 = 0
@@ -46,6 +62,75 @@ impl EncodingType {
}
}
fn decode_ascii(input: &[u8]) -> Result<String, KbinError> {
// ASCII only goes up to 0x7F
match input.iter().position(|&ch| ch >= 0x80) {
Some(first_error) => {
Err(format_err!("Invalid ASCII character at index: {}", first_error)
.context(KbinErrorKind::Encoding)
.into())
},
None => {
let output = String::from_utf8(input.to_vec()).context(KbinErrorKind::Utf8)?;
Ok(output)
},
}
}
fn encode_ascii(input: &str) -> Result<Vec<u8>, KbinError> {
// ASCII only goes up to 0x7F
match input.as_bytes().iter().position(|&ch| ch >= 0x80) {
Some(first_error) => {
Err(format_err!("Unrepresentable character found at index: {}", first_error)
.context(KbinErrorKind::Encoding)
.into())
},
None => {
Ok(input.as_bytes().to_vec())
},
}
}
fn decode_with_encoding(encoding: &'static Encoding, input: &[u8]) -> Result<String, KbinError> {
let (output, actual_encoding, character_replaced) = encoding.decode(input);
//eprintln!("character replaced: {}", character_replaced);
if character_replaced {
warn!("Character replacement occured with: {:?}", output);
}
// `EncodingType::SHIFT_JIS` will ignore invalid characters because Konami's
// implementation will include invalid characters.
if encoding != actual_encoding {
Err(format_err!("Another encoding was used to decode the output: {:?}", actual_encoding)
.context(KbinErrorKind::Encoding)
.into())
} else if !character_replaced || encoding == SHIFT_JIS {
Ok(output.into_owned())
} else {
Err(KbinErrorKind::Encoding.into())
}
}
fn encode_with_encoding(encoding: &'static Encoding, input: &str) -> Result<Vec<u8>, KbinError> {
let (output, actual_encoding, had_unmappable_characters) = encoding.encode(input);
eprintln!("actual encoding: {:?}", actual_encoding);
eprintln!("had unmappable characters: {}", had_unmappable_characters);
if encoding != actual_encoding {
Err(format_err!("Another encoding was used to encode the output: {:?}", actual_encoding)
.context(KbinErrorKind::Encoding)
.into())
} else if had_unmappable_characters {
Err(format_err!("had unmappable characters").context(KbinErrorKind::Encoding).into())
} else {
Ok(output.into_owned())
}
}
/// Decode bytes using the encoding definition from the `encoding` crate.
///
/// A `Some` value indicates an encoding should be used from the `encoding`
@@ -54,21 +139,15 @@ impl EncodingType {
/// `EncodingType::SHIFT_JIS` will ignore invalid characters because Konami's
/// implementation will include invalid characters.
pub fn decode_bytes(&self, input: &[u8]) -> Result<String, KbinError> {
let decoder_fail = |e| {
format_err!("{}", e).context(KbinErrorKind::Encoding)
};
let result = match *self {
match *self {
EncodingType::None |
EncodingType::UTF_8 => String::from_utf8(input.to_vec())?,
EncodingType::UTF_8 => String::from_utf8(input.to_vec()).map_err(KbinError::from),
EncodingType::ASCII => ASCII.decode(input, DecoderTrap::Strict).map_err(decoder_fail)?,
EncodingType::ISO_8859_1 => ISO_8859_1.decode(input, DecoderTrap::Strict).map_err(decoder_fail)?,
EncodingType::EUC_JP => EUC_JP.decode(input, DecoderTrap::Strict).map_err(decoder_fail)?,
EncodingType::SHIFT_JIS => WINDOWS_31J.decode(input, DecoderTrap::Ignore).map_err(decoder_fail)?,
};
Ok(result)
EncodingType::ASCII => Self::decode_ascii(input),
EncodingType::ISO_8859_1 => Self::decode_with_encoding(WINDOWS_1252, input),
EncodingType::EUC_JP => Self::decode_with_encoding(EUC_JP, input),
EncodingType::SHIFT_JIS => Self::decode_with_encoding(SHIFT_JIS, input),
}
}
/// Encode bytes using the encoding definition from the `encoding` crate.
@@ -76,21 +155,18 @@ impl EncodingType {
/// A `Some` value indicates the encoding should be used from the `encoding`
/// crate. A `None` value indicates Rust's own UTF-8 handling should be used.
pub fn encode_bytes(&self, input: &str) -> Result<Vec<u8>, KbinError> {
let encoder_fail = |e| {
format_err!("{}", e).context(KbinErrorKind::Encoding)
};
let mut result = match *self {
EncodingType::None |
EncodingType::UTF_8 => input.as_bytes().to_vec(),
EncodingType::ASCII => ASCII.encode(input, EncoderTrap::Strict).map_err(encoder_fail)?,
EncodingType::ISO_8859_1 => ISO_8859_1.encode(input, EncoderTrap::Strict).map_err(encoder_fail)?,
EncodingType::EUC_JP => EUC_JP.encode(input, EncoderTrap::Strict).map_err(encoder_fail)?,
EncodingType::SHIFT_JIS => WINDOWS_31J.encode(input, EncoderTrap::Strict).map_err(encoder_fail)?,
EncodingType::ASCII => Self::encode_ascii(input)?,
EncodingType::ISO_8859_1 => Self::encode_with_encoding(WINDOWS_1252, input)?,
EncodingType::EUC_JP => Self::encode_with_encoding(EUC_JP, input)?,
EncodingType::SHIFT_JIS => Self::encode_with_encoding(SHIFT_JIS, input)?,
};
// Add trailing null byte
result.reserve_exact(1);
result.push(0);
Ok(result)

View File

@@ -34,12 +34,6 @@ pub enum KbinErrorKind {
#[fail(display = "Unable to read data size")]
DataReadSize,
#[fail(display = "Unable to read 1 byte data")]
DataReadOneByte,
#[fail(display = "Unable to read 2 byte data")]
DataReadTwoByte,
#[fail(display = "Unable to read aligned data from data buffer")]
DataReadAligned,
@@ -76,6 +70,9 @@ pub enum KbinErrorKind {
#[fail(display = "Unable to write sixbit string content")]
SixbitWrite,
#[fail(display = "No node collection found")]
NoNodeCollection,
#[fail(display = "Unable to interpret string as UTF-8")]
Utf8,
@@ -112,7 +109,7 @@ pub enum KbinErrorKind {
#[fail(display = "Invalid input for boolean: {}", _0)]
InvalidBooleanInput(u8),
#[fail(display = "Invalid node type {:?} for operation", _0)]
#[fail(display = "Invalid node type for operation: {:?}", _0)]
InvalidNodeType(StandardType),
#[fail(display = "Invalid state")]

View File

@@ -2,9 +2,10 @@
extern crate byteorder;
extern crate bytes;
extern crate encoding;
extern crate encoding_rs;
extern crate indexmap;
extern crate minidom;
extern crate quick_xml;
extern crate rustc_hex;
#[macro_use] extern crate cfg_if;
@@ -27,12 +28,14 @@ mod options;
mod printer;
mod reader;
mod sixbit;
mod text_reader;
mod to_element;
mod value;
mod writer;
use node::NodeDefinition;
use node_types::StandardType;
use text_reader::TextXmlReader;
// Public exports
pub use compression::Compression;
@@ -142,7 +145,7 @@ fn read_node(reader: &mut Reader, def: NodeDefinition) -> Result<Element> {
Ok(elem)
}
pub fn from_binary(input: &[u8]) -> Result<(Element, EncodingType)> {
pub fn element_from_binary(input: &[u8]) -> Result<(Element, EncodingType)> {
let mut reader = Reader::new(Bytes::from(input))?;
let base = reader.read_node_definition()?;
@@ -152,24 +155,33 @@ pub fn from_binary(input: &[u8]) -> Result<(Element, EncodingType)> {
Ok((elem, encoding))
}
#[inline]
pub fn node_collection_from_slice(input: &[u8]) -> Result<(NodeCollection, EncodingType)> {
node_collection_from_bytes(Bytes::from(input))
}
pub fn node_collection_from_bytes(input: Bytes) -> Result<(NodeCollection, EncodingType)> {
pub fn from_binary(input: Bytes) -> Result<(NodeCollection, EncodingType)> {
let mut reader = Reader::new(input)?;
let collection = NodeCollection::from_iter(&mut reader).ok_or(KbinErrorKind::InvalidState)?;
let collection = NodeCollection::from_iter(&mut reader).ok_or(KbinErrorKind::NoNodeCollection)?;
let encoding = reader.encoding();
Ok((collection, encoding))
}
pub fn node_from_binary(input: Bytes) -> Result<(Node, EncodingType)> {
let (collection, encoding) = node_collection_from_bytes(input)?;
let node = collection.as_node()?;
pub fn from_text_xml(input: &[u8]) -> Result<(NodeCollection, EncodingType)> {
let mut reader = TextXmlReader::new(input);
let collection = reader.as_node_collection()?.ok_or(KbinErrorKind::NoNodeCollection)?;
let encoding = reader.encoding();
Ok((node, encoding))
Ok((collection, encoding))
}
pub fn from_bytes(input: Bytes) -> Result<(NodeCollection, EncodingType)> {
if is_binary_xml(&input) {
from_binary(input)
} else {
from_text_xml(&input)
}
}
#[inline]
pub fn from_slice(input: &[u8]) -> Result<(NodeCollection, EncodingType)> {
from_binary(Bytes::from(input))
}
pub fn to_binary<T>(input: &T) -> Result<Vec<u8>>

View File

@@ -31,7 +31,15 @@ impl NodeCollection {
}
}
pub fn from_iter<I>(mut iter: I) -> Option<NodeCollection>
pub fn with_attributes(base: NodeDefinition, attributes: VecDeque<NodeDefinition>) -> Self {
Self {
base,
attributes,
children: VecDeque::with_capacity(0),
}
}
pub fn from_iter<I>(iter: &mut I) -> Option<NodeCollection>
where I: Iterator<Item = NodeDefinition>
{
let base = if let Some(def) = iter.next() {
@@ -40,10 +48,10 @@ impl NodeCollection {
return None;
};
NodeCollection::with_base(base, &mut iter)
NodeCollection::from_iter_base(base, iter)
}
fn with_base<I>(base: NodeDefinition, iter: &mut I) -> Option<NodeCollection>
fn from_iter_base<I>(base: NodeDefinition, iter: &mut I) -> Option<NodeCollection>
where I: Iterator<Item = NodeDefinition>
{
let mut attributes = VecDeque::new();
@@ -55,7 +63,7 @@ impl NodeCollection {
StandardType::Attribute => attributes.push_back(def),
StandardType::NodeEnd |
StandardType::FileEnd => break,
_ => match NodeCollection::with_base(def, iter) {
_ => match NodeCollection::from_iter_base(def, iter) {
Some(child) => children.push_back(child),
None => return None,
},
@@ -77,6 +85,11 @@ impl NodeCollection {
&self.base
}
#[inline]
pub fn base_mut(&mut self) -> &mut NodeDefinition {
&mut self.base
}
#[inline]
pub fn attributes(&self) -> &VecDeque<NodeDefinition> {
&self.attributes

View File

@@ -44,10 +44,10 @@ impl Key {
fn to_string(&self) -> Result<String, KbinError> {
match self {
Key::Compressed { ref size, ref data } => {
Ok(Sixbit::unpack(data, *size)?)
Sixbit::unpack(data, *size)
},
Key::Uncompressed { encoding, ref data } => {
Ok(encoding.decode_bytes(data)?)
encoding.decode_bytes(data)
},
}
}
@@ -86,6 +86,16 @@ impl NodeDefinition {
(self.node_type, self.is_array)
}
#[inline]
pub fn data<'a>(&'a self) -> &'a NodeData {
&self.data
}
#[inline]
pub fn data_mut<'a>(&'a mut self) -> &'a mut NodeData {
&mut self.data
}
pub fn key(&self) -> Result<Option<String>, KbinError> {
match self.data {
NodeData::Some { ref key, .. } => key.to_string().map(Some),
@@ -180,7 +190,11 @@ impl PartialEq for Key {
impl fmt::Debug for Key {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if let Ok(key) = self.to_string() {
write!(f, "\"{}\"", key)
let variant = match self {
Key::Compressed { .. } => "Compressed",
Key::Uncompressed { .. } => "Uncompressed",
};
write!(f, "{} {{ \"{}\" }}", variant, key)
} else {
match self {
Key::Compressed { ref size, ref data } => {

View File

@@ -8,7 +8,7 @@ use reader::Reader;
pub struct Printer;
impl Printer {
pub fn run(input: &[u8]) -> Result<()> {
pub fn run(input: &[u8]) -> Result<Option<NodeCollection>> {
let mut reader = Reader::new(Bytes::from(input))?;
let mut nodes = Vec::new();
let mut definitions = Vec::new();
@@ -47,12 +47,12 @@ impl Printer {
};
}
let collection = NodeCollection::from_iter(definitions.into_iter());
let collection = NodeCollection::from_iter(&mut definitions.into_iter());
match collection {
Some(collection) => eprintln!("collection: {:#}", collection),
Some(ref collection) => eprintln!("collection: {:#}", collection),
None => eprintln!("collection: {:?}", collection),
};
Ok(())
Ok(collection)
}
}

212
src/text_reader.rs Normal file
View File

@@ -0,0 +1,212 @@
use std::str;
use bytes::{BufMut, Bytes, BytesMut};
use failure::{Fail, ResultExt};
use quick_xml::Reader;
use quick_xml::events::{BytesStart, BytesText, Event};
use quick_xml::events::attributes::Attributes;
use encoding_type::EncodingType;
use error::{KbinErrorKind, Result};
use node::{Key, NodeData, NodeCollection, NodeDefinition};
use node_types::StandardType;
use value::Value;
pub struct TextXmlReader<'a> {
xml_reader: Reader<&'a [u8]>,
encoding: EncodingType,
stack: Vec<(NodeCollection, usize)>,
}
impl<'a> TextXmlReader<'a> {
pub fn new(input: &'a [u8]) -> Self {
let mut xml_reader = Reader::from_reader(input);
xml_reader.trim_text(true);
Self {
xml_reader,
encoding: EncodingType::UTF_8,
// Most kbinxml files that I (mbilker) have come across do not have too
// many inner layers.
stack: Vec::with_capacity(6),
}
}
#[inline]
pub fn encoding(&self) -> EncodingType {
self.encoding
}
fn parse_attribute(&self, key: &[u8], value: &[u8]) -> Result<NodeDefinition> {
// `Attribute` nodes do not have the `is_array` flag set
let node_type = (StandardType::Attribute, false);
let data = NodeData::Some {
key: Key::Uncompressed {
encoding: self.encoding,
data: Bytes::from(key),
},
value_data: Bytes::from(value),
};
Ok(NodeDefinition::with_data(self.encoding, node_type, data))
}
fn parse_attributes(&self, attrs: Attributes<'a>) -> Result<(StandardType, usize, Vec<NodeDefinition>)> {
let mut node_type = None;
let mut count = 0;
let mut attributes = Vec::new();
for attr in attrs {
match attr {
Ok(attr) => {
let value = match attr.unescaped_value() {
Ok(v) => v,
Err(e) => {
error!("Error decoding attribute value: {:?}", e);
attr.value.clone()
},
};
if attr.key == b"__type" {
let value = str::from_utf8(&*value).context(KbinErrorKind::Utf8)?;
node_type = Some(StandardType::from_name(value));
} else if attr.key == b"__count" {
let value = str::from_utf8(&*value).context(KbinErrorKind::Utf8)?;
let num_count = value.parse::<u32>().context(KbinErrorKind::StringParse("array count"))?;
count = num_count as usize;
} else if attr.key == b"__size" {
//let value = str::from_utf8(&*value).context(KbinErrorKind::Utf8)?;
} else {
let definition = self.parse_attribute(attr.key, &value)?;
attributes.push(definition);
}
},
Err(e) => {
error!("Error reading attribute: {:?}", e);
},
}
}
let node_type = match node_type {
Some(node_type) => node_type,
None => {
// Default to `NodeStart`, set to `String` if there is a `Event::Text` event before
// the `Event::End` event.
StandardType::NodeStart
},
};
Ok((node_type, count, attributes))
}
fn handle_start(&self, e: BytesStart) -> Result<(NodeCollection, usize)> {
let (node_type, count, attributes) = self.parse_attributes(e.attributes())?;
let node_type = (node_type, count > 0);
let data = NodeData::Some {
key: Key::Uncompressed {
encoding: self.encoding,
data: Bytes::from(e.name()),
},
// Stub the value for now, handle with `Event::Text`
value_data: Bytes::new(),
};
let base = NodeDefinition::with_data(self.encoding, node_type, data);
let collection = NodeCollection::with_attributes(base, attributes.into());
Ok((collection, count))
}
fn handle_text(event: BytesText, definition: &mut NodeDefinition, count: usize) -> Result<()> {
let data = event.unescaped().context(KbinErrorKind::Utf8)?;
let data = match definition.node_type {
StandardType::String |
StandardType::NodeStart => {
let mut data = BytesMut::from(data.into_owned());
// Add the trailing null byte that kbin has at the end of strings
data.reserve(1);
data.put_u8(0);
data.freeze()
},
_ => {
let text = str::from_utf8(&*data).context(KbinErrorKind::Utf8)?;
let value = Value::from_string(definition.node_type, text, definition.is_array, count)?;
Bytes::from(value.to_bytes()?)
},
};
if definition.node_type == StandardType::NodeStart {
definition.node_type = StandardType::String;
}
if let NodeData::Some { ref mut value_data, .. } = definition.data_mut() {
*value_data = data;
} else {
// There should be a valid `NodeData` structure from the `Event::Start` handler
return Err(KbinErrorKind::InvalidState.into());
}
Ok(())
}
pub fn as_node_collection(&mut self) -> Result<Option<NodeCollection>> {
// A buffer size for reading a `quick_xml::events::Event` that I pulled
// out of my head.
let mut buf = Vec::with_capacity(1024);
loop {
match self.xml_reader.read_event(&mut buf) {
Ok(Event::Start(e)) => {
let start = self.handle_start(e)?;
self.stack.push(start);
},
Ok(Event::Text(e)) => {
if let Some((ref mut collection, ref count)) = self.stack.last_mut() {
let base = collection.base_mut();
Self::handle_text(e, base, *count)?;
}
},
Ok(Event::End(_)) => {
if let Some((collection, _count)) = self.stack.pop() {
if let Some((parent_collection, _count)) = self.stack.last_mut() {
parent_collection.children_mut().push_back(collection);
} else {
// The end of the structure has been reached.
return Ok(Some(collection));
}
}
},
Ok(Event::Empty(e)) => {
let (collection, count) = self.handle_start(e)?;
assert!(count == 0, "empty node should not signal an array");
if let Some((ref mut parent_collection, _count)) = self.stack.last_mut() {
parent_collection.children_mut().push_back(collection);
}
},
Ok(Event::Decl(_)) => {
self.encoding = EncodingType::from_encoding(self.xml_reader.encoding())?;
},
Ok(Event::Eof) => break,
Ok(_) => {},
Err(e) => {
eprintln!("event error: {:?}", e);
return Err(e.context(KbinErrorKind::InvalidState).into())
},
};
buf.clear();
}
Ok(None)
}
}