diff --git a/bemani/format/afp/container.py b/bemani/format/afp/container.py index 35e3d76..309b988 100644 --- a/bemani/format/afp/container.py +++ b/bemani/format/afp/container.py @@ -4,7 +4,7 @@ import struct from PIL import Image from typing import Any, Dict, List, Optional, Tuple -from bemani.format.dxt import DXTBuffer +from bemani.format.tdxt import TDXT from bemani.protocol.binary import BinaryEncoding from bemani.protocol.lz77 import Lz77 from bemani.protocol.node import Node @@ -48,28 +48,56 @@ class Texture: def __init__( self, name: str, - width: int, - height: int, - fmt: int, - header_flags1: int, - header_flags2: int, - header_flags3: int, - fmtflags: int, - rawdata: bytes, + tdxt: TDXT, compressed: Optional[bytes], - imgdata: Any, ) -> None: self.name = name - self.width = width - self.height = height - self.fmt = fmt - self.header_flags1 = header_flags1 - self.header_flags2 = header_flags2 - self.header_flags3 = header_flags3 - self.fmtflags = fmtflags - self.raw = rawdata + self.tdxt = tdxt self.compressed = compressed - self.img = imgdata + + @property + def width(self) -> int: + return self.tdxt.width + + @property + def height(self) -> int: + return self.tdxt.height + + @property + def fmt(self) -> int: + return self.tdxt.fmt + + @property + def fmtflags(self) -> int: + return self.tdxt.fmtflags + + @property + def header_flags1(self) -> int: + return self.tdxt.header_flags1 + + @property + def header_flags2(self) -> int: + return self.tdxt.header_flags2 + + @property + def header_flags3(self) -> int: + return self.tdxt.header_flags3 + + @property + def raw(self) -> bytes: + return self.tdxt.raw + + @property + def img(self) -> Optional[Image.Image]: + return self.tdxt.img + + @img.setter + def img(self, newdata: Image.Image) -> None: + # The TDXT magic container will update the raw for us as well, as long as it's supported. + self.tdxt.img = newdata + + # Unset our cache, so we don't accidentally write the unmodified original data. + self.compressed = None def as_dict(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: return { @@ -472,260 +500,20 @@ class TXP2File(TrackedCoverage, VerboseOutput): ] self.add_coverage(texture_offset, deflated_size + 8) - ( - magic, - header_flags1, - header_flags2, - raw_length, - width, - height, - fmtflags, - expected_zero1, - expected_zero2, - ) = struct.unpack( - f"{self.endian}4sIIIHHIII", - raw_data[0:32], - ) - if raw_length != len(raw_data): - raise Exception("Invalid texture length!") - # I have only ever observed the following values across two different games. - # Don't want to keep the chunk around so let's assert our assumptions. - if (expected_zero1 | expected_zero2) != 0: - raise Exception( - "Found unexpected non-zero value in texture header!" - ) - if raw_data[32:44] != b"\0" * 12: - raise Exception( - "Found unexpected non-zero value in texture header!" - ) - # This is almost ALWAYS 3, but I've seen it be 1 as well, so I guess we have to - # round-trip it if we want to write files back out. I have no clue what it's for. - # I've seen it be 1 only on files used for fonts so far, but I am not sure there - # is any correlation there. - header_flags3 = struct.unpack( - f"{self.endian}I", raw_data[44:48] - )[0] - if raw_data[48:64] != b"\0" * 16: - raise Exception( - "Found unexpected non-zero value in texture header!" - ) - fmt = fmtflags & 0xFF - - # Extract flags that the game cares about. - # flags1 = (fmtflags >> 24) & 0xFF - # flags2 = (fmtflags >> 16) & 0xFF - - # unk1 = 3 if (flags1 & 0xF == 1) else 1 - # unk2 = 3 if ((flags1 >> 4) & 0xF == 1) else 1 - # unk3 = 1 if (flags2 & 0xF == 1) else 2 - # unk4 = 1 if ((flags2 >> 4) & 0xF == 1) else 2 - - if self.endian == "<" and magic != b"TDXT": - raise Exception("Unexpected texture format!") - if self.endian == ">" and magic != b"TXDT": + tdxt = TDXT.fromBytes(raw_data) + if tdxt.endian != self.endian: raise Exception("Unexpected texture format!") - # Since the AFP file format can be found in both big and little endian, its - # possible that some of these loaders might need byteswapping on some platforms. - # This has been tested on files intended for X86 (little endian). I've found that - # the "correct" thing to do is always treat data as little-endian instead of the - # determined endianness of the file. But, this could also be broken per-game, so - # I'm not entirely sure this is fully possible to do generically. - - if fmt == 0x01: - # As far as I can tell, this is 8 bit grayscale. Decoding as such results in - # images that are recognizeable and look correct. - img = Image.frombytes( - "L", - (width, height), - raw_data[64:], - "raw", - "L", - ) - elif fmt == 0x0B: - # 16-bit 565 color RGB format. Game references D3D9 texture format 23 (R5G6B5). - newdata = [] - for i in range(width * height): - pixel = struct.unpack( - "> 0) & 0x1F) << 3 - green = ((pixel >> 5) & 0x3F) << 2 - blue = ((pixel >> 11) & 0x1F) << 3 - - # Scale the colors so they fill the entire 8 bit range. - red = red | (red >> 5) - green = green | (green >> 6) - blue = blue | (blue >> 5) - - newdata.append(struct.pack("> 15) & 0x1) != 0 else 0 - red = ((pixel >> 0) & 0x1F) << 3 - green = ((pixel >> 5) & 0x1F) << 3 - blue = ((pixel >> 10) & 0x1F) << 3 - - # Scale the colors so they fill the entire 8 bit range. - red = red | (red >> 5) - green = green | (green >> 5) - blue = blue | (blue >> 5) - - newdata.append( - struct.pack("> 0) & 0xF) << 4 - green = ((pixel >> 4) & 0xF) << 4 - red = ((pixel >> 8) & 0xF) << 4 - alpha = ((pixel >> 12) & 0xF) << 4 - - # Scale the colors so they fill the entire 8 bit range. - red = red | (red >> 4) - green = green | (green >> 4) - blue = blue | (blue >> 4) - alpha = alpha | (alpha >> 4) - - newdata.append( - struct.pack("": - magic = b"TXDT" - else: - raise Exception("Unexpected texture format!") - - fmtflags = (texture.fmtflags & 0xFFFFFF00) | (texture.fmt & 0xFF) - - raw_texture = ( - struct.pack( - f"{self.endian}4sIIIHHIII", - magic, - texture.header_flags1, - texture.header_flags2, - 64 + len(texture.raw), - texture.width, - texture.height, - fmtflags, - 0, - 0, - ) - + (b"\0" * 12) - + struct.pack( - f"{self.endian}I", - texture.header_flags3, - ) - + (b"\0" * 16) - + texture.raw - ) + raw_texture = texture.tdxt.toBytes() if self.legacy_lz: raise Exception("We don't support legacy lz mode!") @@ -1714,13 +1473,10 @@ class TXP2File(TrackedCoverage, VerboseOutput): if img.width != texture.width or img.height != texture.height: raise Exception("Cannot update texture with different size!") - # Now, get the raw image data. + # Now, get the raw image data, and let the TDXT container refresh the raw. img = img.convert("RGBA") texture.img = img - # Now, refresh the raw texture data for when we write it out. - self._refresh_texture(texture) - return else: raise Exception(f"There is no texture named {name}!") @@ -1753,69 +1509,7 @@ class TXP2File(TrackedCoverage, VerboseOutput): # Now, copy the data over and update the raw texture. for tex in self.textures: if tex.name == texture: - tex.img.paste(sprite_img, (region.left // 2, region.top // 2)) - - # Now, refresh the texture so when we save the file its updated. - self._refresh_texture(tex) - - def _refresh_texture(self, texture: Texture) -> None: - if texture.fmt == 0x0B: - # 16-bit 565 color RGB format. - texture.raw = b"".join( - struct.pack( - f"{self.endian}H", - ( - (((pixel[0] >> 3) & 0x1F) << 11) - | (((pixel[1] >> 2) & 0x3F) << 5) - | ((pixel[2] >> 3) & 0x1F) - ), - ) - for pixel in texture.img.getdata() - ) - elif texture.fmt == 0x13: - # 16-bit A1R5G55 texture format. - texture.raw = b"".join( - struct.pack( - f"{self.endian}H", - ( - (0x8000 if pixel[3] >= 128 else 0x0000) - | (((pixel[0] >> 3) & 0x1F) << 10) - | (((pixel[1] >> 3) & 0x1F) << 5) - | ((pixel[2] >> 3) & 0x1F) - ), - ) - for pixel in texture.img.getdata() - ) - elif texture.fmt == 0x1F: - # 16-bit 4-4-4-4 RGBA format. - texture.raw = b"".join( - struct.pack( - f"{self.endian}H", - ( - ((pixel[2] >> 4) & 0xF) - | (((pixel[1] >> 4) & 0xF) << 4) - | (((pixel[0] >> 4) & 0xF) << 8) - | (((pixel[3] >> 4) & 0xF) << 12) - ), - ) - for pixel in texture.img.getdata() - ) - elif texture.fmt == 0x20: - # 32-bit RGBA format - texture.raw = b"".join( - struct.pack( - f"{self.endian}BBBB", - pixel[2], - pixel[1], - pixel[0], - pixel[3], - ) - for pixel in texture.img.getdata() - ) - else: - raise Exception( - f"Unsupported format {hex(texture.fmt)} for texture {texture.name}" - ) - - # Make sure we don't use the old compressed data. - texture.compressed = None + # Now, composite and refresh the texture so when we save the file its updated. + img = tex.img + img.paste(sprite_img, (region.left // 2, region.top // 2)) + tex.img = img diff --git a/bemani/format/tdxt.py b/bemani/format/tdxt.py new file mode 100644 index 0000000..ebacf8c --- /dev/null +++ b/bemani/format/tdxt.py @@ -0,0 +1,402 @@ +import struct +from PIL import Image +from typing import Optional + +from bemani.format.dxt import DXTBuffer + + +class TDXT: + def __init__( + self, + header_flags1: int, + header_flags2: int, + header_flags3: int, + width: int, + height: int, + fmt: int, + fmtflags: int, + endian: str, + raw: bytes, + img: Optional[Image.Image], + ) -> None: + self.header_flags1 = header_flags1 + self.header_flags2 = header_flags2 + self.header_flags3 = header_flags3 + self.width = width + self.height = height + self.fmt = fmt + self.fmtflags = fmtflags + self.endian = endian + self.__raw = raw + self.__img = img + + @property + def raw(self) -> bytes: + return self.__raw + + @raw.setter + def raw(self, newdata: bytes) -> None: + self.__raw = newdata + newimg = self._rawToImg(self.width, self.height, self.fmt, self.endian, newdata) + width, height = newimg.size + if width != self.width or height != self.height: + raise Exception("Unsupported texture resize operation for TDXT file!") + self.__img = newimg + + @property + def img(self) -> Optional[Image.Image]: + return self.__img + + @img.setter + def img(self, newimg: Image.Image) -> None: + self.__img = newimg + self.__raw = self._imgToRaw(newimg) + + @staticmethod + def fromBytes(raw_data: bytes) -> "TDXT": + # First, check the endianness. + (magic,) = struct.unpack_from("4s", raw_data) + + if magic == b"TDXT": + endian = "<" + elif magic == b"TXDT": + endian = ">" + else: + raise Exception("Unexpected texture format!") + + ( + magic, + header_flags1, + header_flags2, + raw_length, + width, + height, + fmtflags, + expected_zero1, + expected_zero2, + ) = struct.unpack( + f"{endian}4sIIIHHIII", + raw_data[0:32], + ) + if raw_length != len(raw_data): + raise Exception("Invalid texture length!") + + # I have only ever observed the following values across two different games. + # Don't want to keep the chunk around so let's assert our assumptions. + if (expected_zero1 | expected_zero2) != 0: + raise Exception("Found unexpected non-zero value in texture header!") + if raw_data[32:44] != b"\0" * 12: + raise Exception("Found unexpected non-zero value in texture header!") + + # This is almost ALWAYS 3, but I've seen it be 1 as well, so I guess we have to + # round-trip it if we want to write files back out. I have no clue what it's for. + # I've seen it be 1 only on files used for fonts so far, but I am not sure there + # is any correlation there. + header_flags3 = struct.unpack(f"{endian}I", raw_data[44:48])[0] + if raw_data[48:64] != b"\0" * 16: + raise Exception("Found unexpected non-zero value in texture header!") + fmt = fmtflags & 0xFF + + # Extract flags that the game cares about. + # flags1 = (fmtflags >> 24) & 0xFF + # flags2 = (fmtflags >> 16) & 0xFF + + # unk1 = 3 if (flags1 & 0xF == 1) else 1 + # unk2 = 3 if ((flags1 >> 4) & 0xF == 1) else 1 + # unk3 = 1 if (flags2 & 0xF == 1) else 2 + # unk4 = 1 if ((flags2 >> 4) & 0xF == 1) else 2 + + # Convert texture to image if possible, create structure. + return TDXT( + header_flags1=header_flags1, + header_flags2=header_flags2, + header_flags3=header_flags3, + width=width, + height=height, + fmt=fmt, + fmtflags=fmtflags & 0xFFFFFF00, + endian=endian, + raw=raw_data[64:], + img=TDXT._rawToImg(width, height, fmt, endian, raw_data[64:]), + ) + + @staticmethod + def _rawToImg( + width: int, height: int, fmt: int, endian: str, raw_data: bytes + ) -> Optional[Image.Image]: + # Since the AFP file format can be found in both big and little endian, its + # possible that some of these loaders might need byteswapping on some platforms. + # This has been tested on files intended for X86 (little endian) as well as PS3 + # (big endian). I've found that the "correct" thing to do is always treat data as + # little-endian instead of the determined endianness of the file. But, this could + # also be broken per-game, so I'm not entirely sure this is fully possible to do + # generically. However, what's here has been tested across a broad range of games + # and does seem to work. + + if fmt == 0x01: + # As far as I can tell, this is 8 bit grayscale. Decoding as such results in + # images that are recognizeable and look correct. + img = Image.frombytes( + "L", + (width, height), + raw_data, + "raw", + "L", + ) + elif fmt == 0x0B: + # 16-bit 565 color RGB format. Game references D3D9 texture format 23 (R5G6B5). + newdata = [] + for i in range(width * height): + pixel = struct.unpack( + "> 0) & 0x1F) << 3 + green = ((pixel >> 5) & 0x3F) << 2 + blue = ((pixel >> 11) & 0x1F) << 3 + + # Scale the colors so they fill the entire 8 bit range. + red = red | (red >> 5) + green = green | (green >> 6) + blue = blue | (blue >> 5) + + newdata.append(struct.pack("> 15) & 0x1) != 0 else 0 + red = ((pixel >> 0) & 0x1F) << 3 + green = ((pixel >> 5) & 0x1F) << 3 + blue = ((pixel >> 10) & 0x1F) << 3 + + # Scale the colors so they fill the entire 8 bit range. + red = red | (red >> 5) + green = green | (green >> 5) + blue = blue | (blue >> 5) + + newdata.append(struct.pack("> 0) & 0xF) << 4 + green = ((pixel >> 4) & 0xF) << 4 + red = ((pixel >> 8) & 0xF) << 4 + alpha = ((pixel >> 12) & 0xF) << 4 + + # Scale the colors so they fill the entire 8 bit range. + red = red | (red >> 4) + green = green | (green >> 4) + blue = blue | (blue >> 4) + alpha = alpha | (alpha >> 4) + + newdata.append(struct.pack(" bytes: + # Construct the TDXT texture format from our parsed results. + if self.endian == "<": + magic = b"TDXT" + elif self.endian == ">": + magic = b"TXDT" + else: + raise Exception("Unexpected texture format!") + + fmtflags = (self.fmtflags & 0xFFFFFF00) | (self.fmt & 0xFF) + + return ( + struct.pack( + f"{self.endian}4sIIIHHIII", + magic, + self.header_flags1, + self.header_flags2, + 64 + len(self.raw), + self.width, + self.height, + fmtflags, + 0, + 0, + ) + + (b"\0" * 12) + + struct.pack( + f"{self.endian}I", + self.header_flags3, + ) + + (b"\0" * 16) + + self.raw + ) + + def _imgToRaw(self, imgdata: Image.Image) -> bytes: + width, height = imgdata.size + if width != self.width or height != self.height: + raise Exception("Unsupported texture resize operation for TDXT file!") + + if self.fmt == 0x0B: + # 16-bit 565 color RGB format. + raw = b"".join( + struct.pack( + "> 3) & 0x1F) << 11) + | (((pixel[1] >> 2) & 0x3F) << 5) + | ((pixel[2] >> 3) & 0x1F) + ), + ) + for pixel in imgdata.getdata() + ) + elif self.fmt == 0x13: + # 16-bit A1R5G55 texture format. + raw = b"".join( + struct.pack( + "= 128 else 0x0000) + | (((pixel[0] >> 3) & 0x1F) << 10) + | (((pixel[1] >> 3) & 0x1F) << 5) + | ((pixel[2] >> 3) & 0x1F) + ), + ) + for pixel in imgdata.getdata() + ) + elif self.fmt == 0x1F: + # 16-bit 4-4-4-4 RGBA format. + raw = b"".join( + struct.pack( + "> 4) & 0xF) + | (((pixel[1] >> 4) & 0xF) << 4) + | (((pixel[0] >> 4) & 0xF) << 8) + | (((pixel[3] >> 4) & 0xF) << 12) + ), + ) + for pixel in imgdata.getdata() + ) + elif self.fmt == 0x20: + # 32-bit RGBA format, stored in BGRA order. + raw = b"".join( + struct.pack( + "