From 3aff2890c3fd779ef2fb2db1e890077b4ff90b92 Mon Sep 17 00:00:00 2001 From: Will Toohey Date: Sun, 26 Apr 2026 00:23:21 +1000 Subject: [PATCH] (vibe coded) remove cache functionality as repack is now fast enough --- src/ifstools/handlers/generic_file.py | 7 - src/ifstools/handlers/image_file.py | 284 +++++++++++--------------- src/ifstools/ifs.py | 12 +- src/ifstools/ifstools.py | 7 +- 4 files changed, 135 insertions(+), 175 deletions(-) diff --git a/src/ifstools/handlers/generic_file.py b/src/ifstools/handlers/generic_file.py index 50f122f..c17172f 100644 --- a/src/ifstools/handlers/generic_file.py +++ b/src/ifstools/handlers/generic_file.py @@ -46,13 +46,6 @@ class GenericFile(Node): self.size = len(ret) return ret - @property - def needs_preload(self): - return False - - def preload(self, **kwargs): - pass - def repack(self, manifest, data_blob, tqdm_progress, **kwargs): if tqdm_progress: tqdm_progress.write(self.full_path) diff --git a/src/ifstools/handlers/image_file.py b/src/ifstools/handlers/image_file.py index e606f2d..c08a8b0 100644 --- a/src/ifstools/handlers/image_file.py +++ b/src/ifstools/handlers/image_file.py @@ -1,161 +1,123 @@ -import errno -import functools -import time -import timeit -from io import BytesIO -from os import mkdir, utime -from os.path import dirname, getmtime, isfile, join -from struct import pack, unpack -from typing import cast - -import lxml.etree as etree -from kbinxml import KBinXML -from PIL import Image -from tqdm import tqdm - -from .. import utils -from . import lz77 -from .generic_file import GenericFile -from .image_decoders import cachable_formats, encode_png, image_formats - - -class ImageFile(GenericFile): - def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): - raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic') - - @classmethod - def upgrade_generic(cls, gen_file, image_elem, fmt, compress): - self = gen_file - self.__class__ = cls - - self.format = fmt - self.compress = compress - - # all values are multiplied by 2, odd values have never been seen - self.uvrect = [x//2 for x in self._split_ints(image_elem.find('uvrect').text)] - self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)] - self.img_size = ( - self.imgrect[1]-self.imgrect[0], - self.imgrect[3]-self.imgrect[2] - ) - self.uv_size = ( - self.uvrect[1]-self.uvrect[0], - self.uvrect[3]-self.uvrect[2] - ) - - def extract(self, base, use_cache = True, **kwargs): - GenericFile.extract(self, base, **kwargs) - - if use_cache and self.compress and self.from_ifs and self.format in cachable_formats: - self.write_cache(GenericFile._load_from_ifs(self, **kwargs), base) - - def _load_from_ifs(self, crop_to_uvrect = False, raw_pixels = False, **kwargs): - data = GenericFile._load_from_ifs(self, **kwargs) - - if self.compress == 'avslz': - uncompressed_size = unpack('>I', data[:4])[0] - compressed_size = unpack('>I', data[4:8])[0] - # sometimes the headers are missing: not actually compressed - # The 2 extra u32 are moved to the end of the file - # Quality file format. - if len(data) == compressed_size + 8: - data = data[8:] - data = lz77.decompress(data) - assert len(data) == uncompressed_size - else: - data = data[8:] + data[:8] - - if self.format in image_formats: - decoder = image_formats[self.format]['decoder'] - im = decoder(self, data) - else: - raise NotImplementedError('Unknown format {}'.format(self.format)) - - if crop_to_uvrect: - start_x = self.uvrect[0] - self.imgrect[0] - start_y = self.uvrect[2] - self.imgrect[2] - dims = ( - start_x, - start_y, - start_x + self.uv_size[0], - start_y + self.uv_size[1], - ) - im = im.crop(dims) - - if raw_pixels: - return (im.width, im.height), im.tobytes() - else: - return encode_png(im) - - def repack(self, manifest, data_blob, tqdm_progress, **kwargs): - if tqdm_progress: - tqdm_progress.write(self.full_path) - tqdm_progress.update(1) - - if self.compress == 'avslz': - data = self.read_cache() - else: - data = self._load_im() - - # offset, size, timestamp - elem = etree.SubElement(manifest, self.packed_name) - elem.attrib['__type'] = '3s32' - elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) - data_blob.write(data) - # 16 byte alignment - align = len(data) % 16 - if align: - data_blob.write(b'\0' * (16-align)) - - def _load_im(self): - data = self.load() - - im = Image.open(BytesIO(data)) - if im.mode != 'RGBA': - im = im.convert('RGBA') - - if self.format in image_formats: - encoder = image_formats[self.format]['encoder'] - if encoder is None: - # everything else becomes argb8888rev - encoder = image_formats['argb8888rev']['encoder'] - data = encoder(self, im) - else: - raise NotImplementedError('Unknown format {}'.format(self.format)) - - return data - - @property - def needs_preload(self): - cache = join(dirname(self.disk_path), '_cache', self._packed_name) - if isfile(cache): - mtime = int(getmtime(cache)) - if self.time <= mtime: - return False - return True - - def preload(self, use_cache = True, tex_suffix = None, **kwargs): - if not self.needs_preload and use_cache: - return - # Not cached/out of date, compressing - data = self._load_im() - uncompressed_size = len(data) - data = lz77.compress(data) - compressed_size = len(data) - data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data - self.write_cache(data) - - def write_cache(self, data, base = None): - if not self.from_ifs: - base = self.base_path - cache = join(base, self.path, '_cache', self._packed_name) - utils.mkdir_silent(dirname(cache)) - with open(cache, 'wb') as f: - f.write(data) - utime(cache, (self.time, self.time)) - - def read_cache(self): - cache = join(dirname(self.disk_path), '_cache', self._packed_name) - with open(cache, 'rb') as f: - return f.read() - +from io import BytesIO +from struct import pack, unpack + +import lxml.etree as etree +from PIL import Image + +from . import lz77 +from .generic_file import GenericFile +from .image_decoders import encode_png, image_formats + + +class ImageFile(GenericFile): + def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): + raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic') + + @classmethod + def upgrade_generic(cls, gen_file, image_elem, fmt, compress): + self = gen_file + self.__class__ = cls + + self.format = fmt + self.compress = compress + + # all values are multiplied by 2, odd values have never been seen + self.uvrect = [x//2 for x in self._split_ints(image_elem.find('uvrect').text)] + self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)] + self.img_size = ( + self.imgrect[1]-self.imgrect[0], + self.imgrect[3]-self.imgrect[2] + ) + self.uv_size = ( + self.uvrect[1]-self.uvrect[0], + self.uvrect[3]-self.uvrect[2] + ) + + def _load_from_ifs(self, crop_to_uvrect = False, raw_pixels = False, **kwargs): + data = GenericFile._load_from_ifs(self, **kwargs) + + if self.compress == 'avslz': + uncompressed_size = unpack('>I', data[:4])[0] + compressed_size = unpack('>I', data[4:8])[0] + # sometimes the headers are missing: not actually compressed + # The 2 extra u32 are moved to the end of the file + # Quality file format. + if len(data) == compressed_size + 8: + data = data[8:] + data = lz77.decompress(data) + assert len(data) == uncompressed_size + else: + data = data[8:] + data[:8] + + if self.format in image_formats: + decoder = image_formats[self.format]['decoder'] + im = decoder(self, data) + else: + raise NotImplementedError('Unknown format {}'.format(self.format)) + + if crop_to_uvrect: + start_x = self.uvrect[0] - self.imgrect[0] + start_y = self.uvrect[2] - self.imgrect[2] + dims = ( + start_x, + start_y, + start_x + self.uv_size[0], + start_y + self.uv_size[1], + ) + im = im.crop(dims) + + if raw_pixels: + return (im.width, im.height), im.tobytes() + else: + return encode_png(im) + + def _build_packed(self): + data = self._load_im() + if self.compress == 'avslz': + uncompressed_size = len(data) + compressed = lz77.compress(data) + data = pack('>I', uncompressed_size) + pack('>I', len(compressed)) + compressed + return data + + def preload(self, **kwargs): + # Compress in parallel; the actual write loop in repack() runs serially. + self._packed = self._build_packed() + + def repack(self, manifest, data_blob, tqdm_progress, **kwargs): + if tqdm_progress: + tqdm_progress.write(self.full_path) + tqdm_progress.update(1) + + data = getattr(self, '_packed', None) + if data is None: + data = self._build_packed() + + # offset, size, timestamp + elem = etree.SubElement(manifest, self.packed_name) + elem.attrib['__type'] = '3s32' + elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) + data_blob.write(data) + # 16 byte alignment + align = len(data) % 16 + if align: + data_blob.write(b'\0' * (16-align)) + + self._packed = None + + def _load_im(self): + data = self.load() + + im = Image.open(BytesIO(data)) + if im.mode != 'RGBA': + im = im.convert('RGBA') + + if self.format in image_formats: + encoder = image_formats[self.format]['encoder'] + if encoder is None: + # everything else becomes argb8888rev + encoder = image_formats['argb8888rev']['encoder'] + data = encoder(self, im) + else: + raise NotImplementedError('Unknown format {}'.format(self.format)) + + return data diff --git a/src/ifstools/ifs.py b/src/ifstools/ifs.py index c191951..29a07de 100644 --- a/src/ifstools/ifs.py +++ b/src/ifstools/ifs.py @@ -131,8 +131,6 @@ class IFS: extract_manifest = False, path = None, rename_dupes = False, **kwargs): if path is None: path = self.folder_out - if tex_only: - kwargs['use_cache'] = False utils.mkdir_silent(path) utime(path, (self.time, self.time)) @@ -254,12 +252,14 @@ class IFS: def _repack_tree(self, progress = True, **kwargs): files = self.tree.all_files - needs_preload = [f for f in files if f.needs_preload or not kwargs['use_cache']] + to_compress = [f for f in files if isinstance(f, ImageFile)] - # LZ77 compress and PIL decode both release the GIL, so threads scale. + # PNG decode (PIL) and LZ77 compress (Rust) both release the GIL, so + # threads scale. The actual write loop is serial; this stages each + # file's packed bytes in memory. with ThreadPoolExecutor() as ex: - futures = {ex.submit(f.preload, **kwargs): f for f in needs_preload} - with tqdm(total=len(needs_preload), desc='Caching', disable=not progress) as bar: + futures = {ex.submit(f.preload, **kwargs): f for f in to_compress} + with tqdm(total=len(to_compress), desc='Compressing', disable=not progress) as bar: for fut in as_completed(futures): fut.result() f = futures[fut] diff --git a/src/ifstools/ifstools.py b/src/ifstools/ifstools.py index 18f7ce1..1b5e42d 100644 --- a/src/ifstools/ifstools.py +++ b/src/ifstools/ifstools.py @@ -37,7 +37,8 @@ def main(): parser.add_argument('-c', '--canvas', action='store_true', help='dump the image canvas as defined by the texturelist.xml in _canvas.png', dest='dump_canvas') parser.add_argument('--bounds', action='store_true', help='draw image bounds on the exported canvas in red', dest='draw_bbox') parser.add_argument('--uv', action='store_true', help='crop images to uvrect (usually 1px smaller than imgrect). Forces --tex-only', dest='crop_to_uvrect') - parser.add_argument('--no-cache', action='store_false', help='ignore texture cache, recompress all', dest='use_cache') + parser.add_argument('--no-cache', action='store_true', dest='no_cache_deprecated', + help=argparse.SUPPRESS) parser.add_argument('--rename-dupes', action='store_true', help='if two files have the same name but differing case (A.png vs a.png) rename the second as "a (1).png" to allow both to be extracted on Windows') parser.add_argument('-m', '--extract-manifest', action='store_true', help='extract the IFS manifest for inspection', dest='extract_manifest') @@ -54,6 +55,10 @@ def main(): args = parser.parse_args() + if args.no_cache_deprecated: + print("WARNING: --no-cache is deprecated and has no effect; the texture cache has been removed.") + delattr(args, 'no_cache_deprecated') + if args.crop_to_uvrect: args.tex_only = True