diff --git a/.gitignore b/.gitignore index a65055b..97eb7f1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ ifstools.egg-info/ venv/ ifstools.spec +/ifstools-*/ diff --git a/ifstools/__init__.py b/ifstools/__init__.py deleted file mode 100644 index fcb007a..0000000 --- a/ifstools/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .ifstools import main -from .ifs import IFS -from .handlers import GenericFolder, GenericFile diff --git a/ifstools/handlers/__init__.py b/ifstools/handlers/__init__.py deleted file mode 100644 index 3320ceb..0000000 --- a/ifstools/handlers/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .Node import Node - -from .GenericFile import GenericFile -from .ImageFile import ImageFile - -from .GenericFolder import GenericFolder -from .MD5Folder import MD5Folder -from .AfpFolder import AfpFolder -from .TexFolder import TexFolder, ImageCanvas diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ed873ea --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ + +[project] +name = "ifstools" +version = "1.5" +description = "Extractor/repacker for Konmai IFS files" +readme = "README.md" +authors = [ + {name = "mon", email = "me@mon.im"}, +] +dependencies = [ + "kbinxml>=1.5", + "lxml", + "pillow", + "tqdm", +] +requires-python = ">=3.10" + +[project.urls] +Homepage = "https://github.com/mon/ifstools/" + +[project.scripts] +ifstools = "ifstools:main" + +[build-system] +requires = ["setuptools>=78"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e57c90a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -lxml -tqdm -pillow -future -kbinxml>=1.5 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 224a779..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 71278c8..0000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -from setuptools import setup -import sys - - -requires = [ - 'lxml', - 'tqdm', - 'pillow', - 'kbinxml>=1.4', -] -if sys.version_info < (3,0): - requires.append('future') - -version = '1.15' -setup( - name='ifstools', - description='Extractor/repacker for Konmai IFS files', - long_description='See Github for up to date documentation', - version=version, - entry_points = { - 'console_scripts': ['ifstools=ifstools:main'], - }, - packages=['ifstools', 'ifstools.handlers'], - url='https://github.com/mon/ifstools/', - download_url = 'https://github.com/mon/ifstools/archive/{}.tar.gz'.format(version), - author='mon', - author_email='me@mon.im', - install_requires=requires, -) diff --git a/src/ifstools/__init__.py b/src/ifstools/__init__.py new file mode 100644 index 0000000..ca27346 --- /dev/null +++ b/src/ifstools/__init__.py @@ -0,0 +1,2 @@ +from .ifs import IFS +from .ifstools import main diff --git a/ifstools/handlers/AfpFolder.py b/src/ifstools/handlers/afp_folder.py similarity index 95% rename from ifstools/handlers/AfpFolder.py rename to src/ifstools/handlers/afp_folder.py index 1bf06f7..66c44b6 100644 --- a/ifstools/handlers/AfpFolder.py +++ b/src/ifstools/handlers/afp_folder.py @@ -1,4 +1,5 @@ -from . import MD5Folder +from .md5_folder import MD5Folder + class AfpFolder(MD5Folder): diff --git a/ifstools/handlers/GenericFile.py b/src/ifstools/handlers/generic_file.py similarity index 99% rename from ifstools/handlers/GenericFile.py rename to src/ifstools/handlers/generic_file.py index 3901f1c..50f122f 100644 --- a/ifstools/handlers/GenericFile.py +++ b/src/ifstools/handlers/generic_file.py @@ -1,10 +1,11 @@ import os -from kbinxml import KBinXML import lxml.etree as etree +from kbinxml import KBinXML -from .Node import Node from .. import utils +from .node import Node + class GenericFile(Node): def from_xml(self, element): diff --git a/ifstools/handlers/GenericFolder.py b/src/ifstools/handlers/generic_folder.py similarity index 96% rename from ifstools/handlers/GenericFolder.py rename to src/ifstools/handlers/generic_folder.py index acf88ef..d659f11 100644 --- a/ifstools/handlers/GenericFolder.py +++ b/src/ifstools/handlers/generic_folder.py @@ -1,12 +1,13 @@ -from itertools import chain -from os.path import getmtime, basename, dirname, join, realpath, isfile from collections import OrderedDict +from itertools import chain +from os.path import basename, dirname, getmtime, isfile, join, realpath import lxml.etree as etree from tqdm import tqdm -from . import GenericFile -from .Node import Node +from .generic_file import GenericFile +from .node import Node + class GenericFolder(Node): @@ -14,7 +15,8 @@ class GenericFolder(Node): supers = None, super_disable = False, super_skip_bad = False, super_abort_if_bad = False): # circular dependencies mean we import here - from . import AfpFolder, TexFolder + from .afp_folder import AfpFolder + from .tex_folder import TexFolder self.folder_handlers = { 'afp' : AfpFolder, 'tex' : TexFolder, diff --git a/ifstools/handlers/ImageDecoders.py b/src/ifstools/handlers/image_decoders.py similarity index 100% rename from ifstools/handlers/ImageDecoders.py rename to src/ifstools/handlers/image_decoders.py diff --git a/ifstools/handlers/ImageFile.py b/src/ifstools/handlers/image_file.py similarity index 87% rename from ifstools/handlers/ImageFile.py rename to src/ifstools/handlers/image_file.py index c9c775a..f3a0f45 100644 --- a/ifstools/handlers/ImageFile.py +++ b/src/ifstools/handlers/image_file.py @@ -1,154 +1,163 @@ -from io import BytesIO -from struct import unpack, pack -from os.path import getmtime, isfile, join, dirname -from os import utime, mkdir -import errno - -from PIL import Image -import lxml.etree as etree -from kbinxml import KBinXML - -from . import GenericFile -from . import lz77 -from .ImageDecoders import image_formats, cachable_formats -from .. import utils - -class ImageFile(GenericFile): - def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): - raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic') - - @classmethod - def upgrade_generic(cls, gen_file, image_elem, fmt, compress): - self = gen_file - self.__class__ = cls - - self.format = fmt - self.compress = compress - - # all values are multiplied by 2, odd values have never been seen - self.uvrect = [x//2 for x in self._split_ints(image_elem.find('uvrect').text)] - self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)] - self.img_size = ( - self.imgrect[1]-self.imgrect[0], - self.imgrect[3]-self.imgrect[2] - ) - self.uv_size = ( - self.uvrect[1]-self.uvrect[0], - self.uvrect[3]-self.uvrect[2] - ) - - def extract(self, base, use_cache = True, **kwargs): - GenericFile.extract(self, base, **kwargs) - - if use_cache and self.compress and self.from_ifs and self.format in cachable_formats: - self.write_cache(GenericFile._load_from_ifs(self, **kwargs), base) - - def _load_from_ifs(self, crop_to_uvrect = False, **kwargs): - data = GenericFile._load_from_ifs(self, **kwargs) - - if self.compress == 'avslz': - uncompressed_size = unpack('>I', data[:4])[0] - compressed_size = unpack('>I', data[4:8])[0] - # sometimes the headers are missing: not actually compressed - # The 2 extra u32 are moved to the end of the file - # Quality file format. - if len(data) == compressed_size + 8: - data = data[8:] - data = lz77.decompress(data) - assert len(data) == uncompressed_size - else: - data = data[8:] + data[:8] - - if self.format in image_formats: - decoder = image_formats[self.format]['decoder'] - im = decoder(self, data) - else: - raise NotImplementedError('Unknown format {}'.format(self.format)) - - if crop_to_uvrect: - start_x = self.uvrect[0] - self.imgrect[0] - start_y = self.uvrect[2] - self.imgrect[2] - dims = ( - start_x, - start_y, - start_x + self.uv_size[0], - start_y + self.uv_size[1], - ) - im = im.crop(dims) - - b = BytesIO() - im.save(b, format = 'PNG') - return b.getvalue() - - def repack(self, manifest, data_blob, tqdm_progress, **kwargs): - if tqdm_progress: - tqdm_progress.write(self.full_path) - tqdm_progress.update(1) - - if self.compress == 'avslz': - data = self.read_cache() - else: - data = self._load_im() - - # offset, size, timestamp - elem = etree.SubElement(manifest, self.packed_name) - elem.attrib['__type'] = '3s32' - elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) - data_blob.write(data) - # 16 byte alignment - align = len(data) % 16 - if align: - data_blob.write(b'\0' * (16-align)) - - def _load_im(self): - data = self.load() - - im = Image.open(BytesIO(data)) - if im.mode != 'RGBA': - im = im.convert('RGBA') - - if self.format in image_formats: - encoder = image_formats[self.format]['encoder'] - if encoder is None: - # everything else becomes argb8888rev - encoder = image_formats['argb8888rev']['encoder'] - data = encoder(self, im) - else: - raise NotImplementedError('Unknown format {}'.format(self.format)) - - return data - - @property - def needs_preload(self): - cache = join(dirname(self.disk_path), '_cache', self._packed_name) - if isfile(cache): - mtime = int(getmtime(cache)) - if self.time <= mtime: - return False - return True - - def preload(self, use_cache = True, tex_suffix = None, **kwargs): - if not self.needs_preload and use_cache: - return - # Not cached/out of date, compressing - data = self._load_im() - uncompressed_size = len(data) - data = lz77.compress(data) - compressed_size = len(data) - data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data - self.write_cache(data) - - def write_cache(self, data, base = None): - if not self.from_ifs: - base = self.base_path - cache = join(base, self.path, '_cache', self._packed_name) - utils.mkdir_silent(dirname(cache)) - with open(cache, 'wb') as f: - f.write(data) - utime(cache, (self.time, self.time)) - - def read_cache(self): - cache = join(dirname(self.disk_path), '_cache', self._packed_name) - with open(cache, 'rb') as f: - return f.read() - +import errno +import functools +import time +import timeit +from io import BytesIO +from os import mkdir, utime +from os.path import dirname, getmtime, isfile, join +from struct import pack, unpack +from typing import cast + +import lxml.etree as etree +from kbinxml import KBinXML +from PIL import Image +from tqdm import tqdm + +from .. import utils +from . import lz77 +from .generic_file import GenericFile +from .image_decoders import cachable_formats, image_formats + + +class ImageFile(GenericFile): + def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): + raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic') + + @classmethod + def upgrade_generic(cls, gen_file, image_elem, fmt, compress): + self = gen_file + self.__class__ = cls + + self.format = fmt + self.compress = compress + + # all values are multiplied by 2, odd values have never been seen + self.uvrect = [x//2 for x in self._split_ints(image_elem.find('uvrect').text)] + self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)] + self.img_size = ( + self.imgrect[1]-self.imgrect[0], + self.imgrect[3]-self.imgrect[2] + ) + self.uv_size = ( + self.uvrect[1]-self.uvrect[0], + self.uvrect[3]-self.uvrect[2] + ) + + def extract(self, base, use_cache = True, **kwargs): + GenericFile.extract(self, base, **kwargs) + + if use_cache and self.compress and self.from_ifs and self.format in cachable_formats: + self.write_cache(GenericFile._load_from_ifs(self, **kwargs), base) + + def _load_from_ifs(self, crop_to_uvrect = False, raw_pixels = False, **kwargs): + data = GenericFile._load_from_ifs(self, **kwargs) + + if self.compress == 'avslz': + uncompressed_size = unpack('>I', data[:4])[0] + compressed_size = unpack('>I', data[4:8])[0] + # sometimes the headers are missing: not actually compressed + # The 2 extra u32 are moved to the end of the file + # Quality file format. + if len(data) == compressed_size + 8: + data = data[8:] + data = lz77.decompress(data) + assert len(data) == uncompressed_size + else: + data = data[8:] + data[:8] + + if self.format in image_formats: + decoder = image_formats[self.format]['decoder'] + im = decoder(self, data) + else: + raise NotImplementedError('Unknown format {}'.format(self.format)) + + if crop_to_uvrect: + start_x = self.uvrect[0] - self.imgrect[0] + start_y = self.uvrect[2] - self.imgrect[2] + dims = ( + start_x, + start_y, + start_x + self.uv_size[0], + start_y + self.uv_size[1], + ) + im = im.crop(dims) + + b = BytesIO() + if raw_pixels: + return (im.width, im.height), im.tobytes() + else: + im.save(b, format = 'PNG') + return b.getvalue() + + def repack(self, manifest, data_blob, tqdm_progress, **kwargs): + if tqdm_progress: + tqdm_progress.write(self.full_path) + tqdm_progress.update(1) + + if self.compress == 'avslz': + data = self.read_cache() + else: + data = self._load_im() + + # offset, size, timestamp + elem = etree.SubElement(manifest, self.packed_name) + elem.attrib['__type'] = '3s32' + elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) + data_blob.write(data) + # 16 byte alignment + align = len(data) % 16 + if align: + data_blob.write(b'\0' * (16-align)) + + def _load_im(self): + data = self.load() + + im = Image.open(BytesIO(data)) + if im.mode != 'RGBA': + im = im.convert('RGBA') + + if self.format in image_formats: + encoder = image_formats[self.format]['encoder'] + if encoder is None: + # everything else becomes argb8888rev + encoder = image_formats['argb8888rev']['encoder'] + data = encoder(self, im) + else: + raise NotImplementedError('Unknown format {}'.format(self.format)) + + return data + + @property + def needs_preload(self): + cache = join(dirname(self.disk_path), '_cache', self._packed_name) + if isfile(cache): + mtime = int(getmtime(cache)) + if self.time <= mtime: + return False + return True + + def preload(self, use_cache = True, tex_suffix = None, **kwargs): + if not self.needs_preload and use_cache: + return + # Not cached/out of date, compressing + data = self._load_im() + uncompressed_size = len(data) + data = lz77.compress(data) + compressed_size = len(data) + data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data + self.write_cache(data) + + def write_cache(self, data, base = None): + if not self.from_ifs: + base = self.base_path + cache = join(base, self.path, '_cache', self._packed_name) + utils.mkdir_silent(dirname(cache)) + with open(cache, 'wb') as f: + f.write(data) + utime(cache, (self.time, self.time)) + + def read_cache(self): + cache = join(dirname(self.disk_path), '_cache', self._packed_name) + with open(cache, 'rb') as f: + return f.read() + diff --git a/ifstools/handlers/lz77.py b/src/ifstools/handlers/lz77.py similarity index 99% rename from ifstools/handlers/lz77.py rename to src/ifstools/handlers/lz77.py index a06418c..75d84aa 100644 --- a/ifstools/handlers/lz77.py +++ b/src/ifstools/handlers/lz77.py @@ -1,7 +1,7 @@ # consistency with py 2/3 from builtins import bytes -from struct import unpack, pack from io import BytesIO +from struct import pack, unpack from tqdm import tqdm diff --git a/ifstools/handlers/MD5Folder.py b/src/ifstools/handlers/md5_folder.py similarity index 97% rename from ifstools/handlers/MD5Folder.py rename to src/ifstools/handlers/md5_folder.py index 0dc8e53..7505a39 100644 --- a/ifstools/handlers/MD5Folder.py +++ b/src/ifstools/handlers/md5_folder.py @@ -2,7 +2,8 @@ from hashlib import md5 from kbinxml import KBinXML -from . import GenericFolder +from .generic_folder import GenericFolder + class MD5Folder(GenericFolder): diff --git a/ifstools/handlers/Node.py b/src/ifstools/handlers/node.py similarity index 100% rename from ifstools/handlers/Node.py rename to src/ifstools/handlers/node.py diff --git a/ifstools/handlers/TexFolder.py b/src/ifstools/handlers/tex_folder.py similarity index 95% rename from ifstools/handlers/TexFolder.py rename to src/ifstools/handlers/tex_folder.py index fed2216..75f3d5c 100644 --- a/ifstools/handlers/TexFolder.py +++ b/src/ifstools/handlers/tex_folder.py @@ -1,11 +1,14 @@ from io import BytesIO from kbinxml import KBinXML -from tqdm import tqdm from PIL import Image, ImageDraw +from tqdm import tqdm + +from .generic_file import GenericFile +from .image_decoders import cachable_formats +from .image_file import ImageFile +from .md5_folder import MD5Folder -from . import MD5Folder, ImageFile, GenericFile -from .ImageDecoders import cachable_formats class TextureList(GenericFile): def _load_from_filesystem(self, **kwargs): diff --git a/ifstools/ifs.py b/src/ifstools/ifs.py similarity index 93% rename from ifstools/ifs.py rename to src/ifstools/ifs.py index fa2cb6c..ff2ba34 100644 --- a/ifstools/ifs.py +++ b/src/ifstools/ifs.py @@ -1,19 +1,22 @@ -from collections import defaultdict -from multiprocessing import Pool -from os.path import basename, dirname, splitext, join, isdir, isfile, getmtime -from os import utime, walk -from io import BytesIO -import itertools import hashlib -import lxml.etree as etree +import itertools +from collections import defaultdict +from io import BytesIO +from multiprocessing import Pool +from os import utime, walk +from os.path import basename, getmtime, isdir, isfile, join, splitext from time import time as unixtime -from tqdm import tqdm +import lxml.etree as etree from kbinxml import KBinXML from kbinxml.bytebuffer import ByteBuffer +from tqdm import tqdm -from .handlers import GenericFolder, MD5Folder, ImageFile, ImageCanvas from . import utils +from .handlers.generic_folder import GenericFolder +from .handlers.image_file import ImageFile +from .handlers.md5_folder import MD5Folder +from .handlers.tex_folder import ImageCanvas SIGNATURE = 0x6CAD8F89 @@ -185,7 +188,7 @@ class IFS: # extract the files for f in tqdm(self.tree.all_files, disable = not progress): # allow recurse + tex_only to extract ifs files - if tex_only and not isinstance(f, ImageFile) and not isinstance(f, ImageCanvas) and not (recurse and f.name.endswith('.ifs')): + if tex_only and not isinstance(f, (ImageFile, ImageCanvas)) and not (recurse and f.name.endswith('.ifs')): continue f.extract(path, **kwargs) if progress: @@ -199,7 +202,7 @@ class IFS: # you can't pickle open files, so this won't work. Perhaps there is a way around it? - '''to_extract = (f for f in self.tree.all_files if not(tex_only and not isinstance(f, ImageFile) and not isinstance(f, ImageCanvas))) + '''to_extract = (f for f in self.tree.all_files if not(tex_only and not isinstance(f, (ImageFile, ImageCanvas)))) p = Pool() args = zip(to_extract, itertools.cycle((path,)), itertools.cycle((kwargs,))) diff --git a/ifstools/ifstools.py b/src/ifstools/ifstools.py similarity index 100% rename from ifstools/ifstools.py rename to src/ifstools/ifstools.py diff --git a/ifstools/utils.py b/src/ifstools/utils.py similarity index 100% rename from ifstools/utils.py rename to src/ifstools/utils.py