Refactor the source tree a bit

.gitignore

@@ -7,3 +7,4 @@ dist/
 ifstools.egg-info/
 venv/
 ifstools.spec
+/ifstools-*/

ifstools/__init__.py

@@ -1,3 +0,0 @@
-from .ifstools import main
-from .ifs import IFS
-from .handlers import GenericFolder, GenericFile

ifstools/handlers/__init__.py

@@ -1,9 +0,0 @@
-from .Node import Node
-
-from .GenericFile import GenericFile
-from .ImageFile import ImageFile
-
-from .GenericFolder import GenericFolder
-from .MD5Folder import MD5Folder
-from .AfpFolder import AfpFolder
-from .TexFolder import TexFolder, ImageCanvas

pyproject.toml

@@ -0,0 +1,26 @@
+
+[project]
+name = "ifstools"
+version = "1.5"
+description = "Extractor/repacker for Konmai IFS files"
+readme = "README.md"
+authors = [
+    {name = "mon", email = "me@mon.im"},
+]
+dependencies = [
+    "kbinxml>=1.5",
+    "lxml",
+    "pillow",
+    "tqdm",
+]
+requires-python = ">=3.10"
+
+[project.urls]
+Homepage = "https://github.com/mon/ifstools/"
+
+[project.scripts]
+ifstools = "ifstools:main"
+
+[build-system]
+requires = ["setuptools>=78"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -1,5 +0,0 @@
-lxml
-tqdm
-pillow
-future
-kbinxml>=1.5

setup.cfg

@@ -1,2 +0,0 @@
-[metadata]
-description-file = README.md

setup.py

@@ -1,29 +0,0 @@
-from setuptools import setup
-import sys
-
-
-requires = [
-        'lxml',
-        'tqdm',
-        'pillow',
-        'kbinxml>=1.4',
-]
-if sys.version_info < (3,0):
-    requires.append('future')
-
-version = '1.15'
-setup(
-    name='ifstools',
-    description='Extractor/repacker for Konmai IFS files',
-    long_description='See Github for up to date documentation',
-    version=version,
-    entry_points = {
-        'console_scripts': ['ifstools=ifstools:main'],
-    },
-    packages=['ifstools', 'ifstools.handlers'],
-    url='https://github.com/mon/ifstools/',
-    download_url = 'https://github.com/mon/ifstools/archive/{}.tar.gz'.format(version),
-    author='mon',
-    author_email='me@mon.im',
-    install_requires=requires,
-)

src/ifstools/__init__.py

@@ -0,0 +1,2 @@
+from .ifs import IFS
+from .ifstools import main

src/ifstools/handlers/afp_folder.py

@@ -1,4 +1,5 @@
-from . import MD5Folder
+from .md5_folder import MD5Folder
+
 
 class AfpFolder(MD5Folder):
 

src/ifstools/handlers/generic_file.py

@@ -1,10 +1,11 @@
 import os
 
-from kbinxml import KBinXML
 import lxml.etree as etree
+from kbinxml import KBinXML
 
-from .Node import Node
 from .. import utils
+from .node import Node
+
 
 class GenericFile(Node):
     def from_xml(self, element):

src/ifstools/handlers/generic_folder.py

@@ -1,12 +1,13 @@
-from itertools import chain
-from os.path import getmtime, basename, dirname, join, realpath, isfile
 from collections import OrderedDict
+from itertools import chain
+from os.path import basename, dirname, getmtime, isfile, join, realpath
 
 import lxml.etree as etree
 from tqdm import tqdm
 
-from . import GenericFile
-from .Node import Node
+from .generic_file import GenericFile
+from .node import Node
+
 
 class GenericFolder(Node):
 
@@ -14,7 +15,8 @@ class GenericFolder(Node):
             supers = None, super_disable = False, super_skip_bad = False,
             super_abort_if_bad = False):
         # circular dependencies mean we import here
-        from . import AfpFolder, TexFolder
+        from .afp_folder import AfpFolder
+        from .tex_folder import TexFolder
         self.folder_handlers = {
             'afp' : AfpFolder,
             'tex' : TexFolder,

src/ifstools/handlers/image_file.py

@@ -1,154 +1,163 @@
-from io import BytesIO
-from struct import unpack, pack
-from os.path import getmtime, isfile, join, dirname
-from os import utime, mkdir
-import errno
-
-from PIL import Image
-import lxml.etree as etree
-from kbinxml import KBinXML
-
-from . import GenericFile
-from . import lz77
-from .ImageDecoders import image_formats, cachable_formats
-from .. import utils
-
-class ImageFile(GenericFile):
-    def __init__(self, ifs_data, obj, parent = None, path = '', name = ''):
-        raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic')
-
-    @classmethod
-    def upgrade_generic(cls, gen_file, image_elem, fmt, compress):
-        self = gen_file
-        self.__class__ = cls
-
-        self.format = fmt
-        self.compress = compress
-
-        # all values are multiplied by 2, odd values have never been seen
-        self.uvrect =  [x//2 for x in self._split_ints(image_elem.find('uvrect').text)]
-        self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)]
-        self.img_size = (
-            self.imgrect[1]-self.imgrect[0],
-            self.imgrect[3]-self.imgrect[2]
-        )
-        self.uv_size = (
-            self.uvrect[1]-self.uvrect[0],
-            self.uvrect[3]-self.uvrect[2]
-        )
-
-    def extract(self, base, use_cache = True, **kwargs):
-        GenericFile.extract(self, base, **kwargs)
-
-        if use_cache and self.compress and self.from_ifs and self.format in cachable_formats:
-            self.write_cache(GenericFile._load_from_ifs(self, **kwargs), base)
-
-    def _load_from_ifs(self, crop_to_uvrect = False, **kwargs):
-        data = GenericFile._load_from_ifs(self, **kwargs)
-
-        if self.compress == 'avslz':
-            uncompressed_size = unpack('>I', data[:4])[0]
-            compressed_size = unpack('>I', data[4:8])[0]
-            # sometimes the headers are missing: not actually compressed
-            # The 2 extra u32 are moved to the end of the file
-            # Quality file format.
-            if len(data) == compressed_size + 8:
-                data = data[8:]
-                data = lz77.decompress(data)
-                assert len(data) == uncompressed_size
-            else:
-                data = data[8:] + data[:8]
-
-        if self.format in image_formats:
-            decoder = image_formats[self.format]['decoder']
-            im = decoder(self, data)
-        else:
-            raise NotImplementedError('Unknown format {}'.format(self.format))
-
-        if crop_to_uvrect:
-            start_x = self.uvrect[0] - self.imgrect[0]
-            start_y = self.uvrect[2] - self.imgrect[2]
-            dims = (
-                start_x,
-                start_y,
-                start_x + self.uv_size[0],
-                start_y + self.uv_size[1],
-            )
-            im = im.crop(dims)
-
-        b = BytesIO()
-        im.save(b, format = 'PNG')
-        return b.getvalue()
-
-    def repack(self, manifest, data_blob, tqdm_progress, **kwargs):
-        if tqdm_progress:
-            tqdm_progress.write(self.full_path)
-            tqdm_progress.update(1)
-
-        if self.compress == 'avslz':
-            data = self.read_cache()
-        else:
-            data = self._load_im()
-
-        # offset, size, timestamp
-        elem = etree.SubElement(manifest, self.packed_name)
-        elem.attrib['__type'] = '3s32'
-        elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time)
-        data_blob.write(data)
-        # 16 byte alignment
-        align = len(data) % 16
-        if align:
-            data_blob.write(b'\0' * (16-align))
-
-    def _load_im(self):
-        data = self.load()
-
-        im = Image.open(BytesIO(data))
-        if im.mode != 'RGBA':
-            im = im.convert('RGBA')
-
-        if self.format in image_formats:
-            encoder = image_formats[self.format]['encoder']
-            if encoder is None:
-                # everything else becomes argb8888rev
-                encoder = image_formats['argb8888rev']['encoder']
-            data = encoder(self, im)
-        else:
-            raise NotImplementedError('Unknown format {}'.format(self.format))
-
-        return data
-
-    @property
-    def needs_preload(self):
-        cache = join(dirname(self.disk_path), '_cache', self._packed_name)
-        if isfile(cache):
-            mtime = int(getmtime(cache))
-            if self.time <= mtime:
-                return False
-        return True
-
-    def preload(self, use_cache = True, tex_suffix = None, **kwargs):
-        if not self.needs_preload and use_cache:
-            return
-        # Not cached/out of date, compressing
-        data = self._load_im()
-        uncompressed_size = len(data)
-        data = lz77.compress(data)
-        compressed_size = len(data)
-        data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data
-        self.write_cache(data)
-
-    def write_cache(self, data, base = None):
-        if not self.from_ifs:
-            base = self.base_path
-        cache = join(base, self.path, '_cache', self._packed_name)
-        utils.mkdir_silent(dirname(cache))
-        with open(cache, 'wb') as f:
-            f.write(data)
-        utime(cache, (self.time, self.time))
-
-    def read_cache(self):
-        cache = join(dirname(self.disk_path), '_cache', self._packed_name)
-        with open(cache, 'rb') as f:
-            return f.read()
-
+import errno
+import functools
+import time
+import timeit
+from io import BytesIO
+from os import mkdir, utime
+from os.path import dirname, getmtime, isfile, join
+from struct import pack, unpack
+from typing import cast
+
+import lxml.etree as etree
+from kbinxml import KBinXML
+from PIL import Image
+from tqdm import tqdm
+
+from .. import utils
+from . import lz77
+from .generic_file import GenericFile
+from .image_decoders import cachable_formats, image_formats
+
+
+class ImageFile(GenericFile):
+    def __init__(self, ifs_data, obj, parent = None, path = '', name = ''):
+        raise Exception('ImageFile must be instantiated from existing GenericFile with ImageFile.upgrade_generic')
+
+    @classmethod
+    def upgrade_generic(cls, gen_file, image_elem, fmt, compress):
+        self = gen_file
+        self.__class__ = cls
+
+        self.format = fmt
+        self.compress = compress
+
+        # all values are multiplied by 2, odd values have never been seen
+        self.uvrect =  [x//2 for x in self._split_ints(image_elem.find('uvrect').text)]
+        self.imgrect = [x//2 for x in self._split_ints(image_elem.find('imgrect').text)]
+        self.img_size = (
+            self.imgrect[1]-self.imgrect[0],
+            self.imgrect[3]-self.imgrect[2]
+        )
+        self.uv_size = (
+            self.uvrect[1]-self.uvrect[0],
+            self.uvrect[3]-self.uvrect[2]
+        )
+
+    def extract(self, base, use_cache = True, **kwargs):
+        GenericFile.extract(self, base, **kwargs)
+
+        if use_cache and self.compress and self.from_ifs and self.format in cachable_formats:
+            self.write_cache(GenericFile._load_from_ifs(self, **kwargs), base)
+
+    def _load_from_ifs(self, crop_to_uvrect = False, raw_pixels = False, **kwargs):
+        data = GenericFile._load_from_ifs(self, **kwargs)
+
+        if self.compress == 'avslz':
+            uncompressed_size = unpack('>I', data[:4])[0]
+            compressed_size = unpack('>I', data[4:8])[0]
+            # sometimes the headers are missing: not actually compressed
+            # The 2 extra u32 are moved to the end of the file
+            # Quality file format.
+            if len(data) == compressed_size + 8:
+                data = data[8:]
+                data = lz77.decompress(data)
+                assert len(data) == uncompressed_size
+            else:
+                data = data[8:] + data[:8]
+
+        if self.format in image_formats:
+            decoder = image_formats[self.format]['decoder']
+            im = decoder(self, data)
+        else:
+            raise NotImplementedError('Unknown format {}'.format(self.format))
+
+        if crop_to_uvrect:
+            start_x = self.uvrect[0] - self.imgrect[0]
+            start_y = self.uvrect[2] - self.imgrect[2]
+            dims = (
+                start_x,
+                start_y,
+                start_x + self.uv_size[0],
+                start_y + self.uv_size[1],
+            )
+            im = im.crop(dims)
+
+        b = BytesIO()
+        if raw_pixels:
+            return (im.width, im.height), im.tobytes()
+        else:
+            im.save(b, format = 'PNG')
+            return b.getvalue()
+
+    def repack(self, manifest, data_blob, tqdm_progress, **kwargs):
+        if tqdm_progress:
+            tqdm_progress.write(self.full_path)
+            tqdm_progress.update(1)
+
+        if self.compress == 'avslz':
+            data = self.read_cache()
+        else:
+            data = self._load_im()
+
+        # offset, size, timestamp
+        elem = etree.SubElement(manifest, self.packed_name)
+        elem.attrib['__type'] = '3s32'
+        elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time)
+        data_blob.write(data)
+        # 16 byte alignment
+        align = len(data) % 16
+        if align:
+            data_blob.write(b'\0' * (16-align))
+
+    def _load_im(self):
+        data = self.load()
+
+        im = Image.open(BytesIO(data))
+        if im.mode != 'RGBA':
+            im = im.convert('RGBA')
+
+        if self.format in image_formats:
+            encoder = image_formats[self.format]['encoder']
+            if encoder is None:
+                # everything else becomes argb8888rev
+                encoder = image_formats['argb8888rev']['encoder']
+            data = encoder(self, im)
+        else:
+            raise NotImplementedError('Unknown format {}'.format(self.format))
+
+        return data
+
+    @property
+    def needs_preload(self):
+        cache = join(dirname(self.disk_path), '_cache', self._packed_name)
+        if isfile(cache):
+            mtime = int(getmtime(cache))
+            if self.time <= mtime:
+                return False
+        return True
+
+    def preload(self, use_cache = True, tex_suffix = None, **kwargs):
+        if not self.needs_preload and use_cache:
+            return
+        # Not cached/out of date, compressing
+        data = self._load_im()
+        uncompressed_size = len(data)
+        data = lz77.compress(data)
+        compressed_size = len(data)
+        data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data
+        self.write_cache(data)
+
+    def write_cache(self, data, base = None):
+        if not self.from_ifs:
+            base = self.base_path
+        cache = join(base, self.path, '_cache', self._packed_name)
+        utils.mkdir_silent(dirname(cache))
+        with open(cache, 'wb') as f:
+            f.write(data)
+        utime(cache, (self.time, self.time))
+
+    def read_cache(self):
+        cache = join(dirname(self.disk_path), '_cache', self._packed_name)
+        with open(cache, 'rb') as f:
+            return f.read()
+

src/ifstools/handlers/lz77.py

@@ -1,7 +1,7 @@
 # consistency with py 2/3
 from builtins import bytes
-from struct import unpack, pack
 from io import BytesIO
+from struct import pack, unpack
 
 from tqdm import tqdm
 

src/ifstools/handlers/md5_folder.py

@@ -2,7 +2,8 @@ from hashlib import md5
 
 from kbinxml import KBinXML
 
-from . import GenericFolder
+from .generic_folder import GenericFolder
+
 
 class MD5Folder(GenericFolder):
 

src/ifstools/handlers/tex_folder.py

@@ -1,11 +1,14 @@
 from io import BytesIO
 
 from kbinxml import KBinXML
-from tqdm import tqdm
 from PIL import Image, ImageDraw
+from tqdm import tqdm
+
+from .generic_file import GenericFile
+from .image_decoders import cachable_formats
+from .image_file import ImageFile
+from .md5_folder import MD5Folder
 
-from . import MD5Folder, ImageFile, GenericFile
-from .ImageDecoders import cachable_formats
 
 class TextureList(GenericFile):
     def _load_from_filesystem(self, **kwargs):

src/ifstools/ifs.py

@@ -1,19 +1,22 @@
-from collections import defaultdict
-from multiprocessing import Pool
-from os.path import basename, dirname, splitext, join, isdir, isfile, getmtime
-from os import utime, walk
-from io import BytesIO
-import itertools
 import hashlib
-import lxml.etree as etree
+import itertools
+from collections import defaultdict
+from io import BytesIO
+from multiprocessing import Pool
+from os import utime, walk
+from os.path import basename, getmtime, isdir, isfile, join, splitext
 from time import time as unixtime
 
-from tqdm import tqdm
+import lxml.etree as etree
 from kbinxml import KBinXML
 from kbinxml.bytebuffer import ByteBuffer
+from tqdm import tqdm
 
-from .handlers import GenericFolder, MD5Folder, ImageFile, ImageCanvas
 from . import utils
+from .handlers.generic_folder import GenericFolder
+from .handlers.image_file import ImageFile
+from .handlers.md5_folder import MD5Folder
+from .handlers.tex_folder import ImageCanvas
 
 SIGNATURE = 0x6CAD8F89
 
@@ -185,7 +188,7 @@ class IFS:
         # extract the files
         for f in tqdm(self.tree.all_files, disable = not progress):
             # allow recurse + tex_only to extract ifs files
-            if tex_only and not isinstance(f, ImageFile) and not isinstance(f, ImageCanvas) and not (recurse and f.name.endswith('.ifs')):
+            if tex_only and not isinstance(f, (ImageFile, ImageCanvas)) and not (recurse and f.name.endswith('.ifs')):
                 continue
             f.extract(path, **kwargs)
             if progress:
@@ -199,7 +202,7 @@ class IFS:
 
 
         # you can't pickle open files, so this won't work. Perhaps there is a way around it?
-        '''to_extract = (f for f in self.tree.all_files if not(tex_only and not isinstance(f, ImageFile) and not isinstance(f, ImageCanvas)))
+        '''to_extract = (f for f in self.tree.all_files if not(tex_only and not isinstance(f, (ImageFile, ImageCanvas))))
 
         p = Pool()
         args = zip(to_extract, itertools.cycle((path,)), itertools.cycle((kwargs,)))