diff --git a/pkNX.Containers/Misc/GFPack.cs b/pkNX.Containers/Misc/GFPack.cs index 6a49e5cd..001255b4 100644 --- a/pkNX.Containers/Misc/GFPack.cs +++ b/pkNX.Containers/Misc/GFPack.cs @@ -16,9 +16,11 @@ public class GFPack : IEnumerable, IFileContainer // Overall structure: Header, metadata, and the raw compressed files public GFPackHeader Header { get; set; } - public FileHashPath[] HashPaths { get; set; } - public FileHashIndex[] HashIndexes { get; set; } + public GFPackPointers Pointers { get; set; } + public FileHashAbsolute[] HashAbsolute { get; set; } + public FileHashFolder[] HashInFolder { get; set; } public FileData[] FileTable { get; set; } + public byte[][] CompressedFiles { get; set; } public byte[][] DecompressedFiles { get; set; } @@ -42,15 +44,27 @@ private void ReadPack(BinaryReader br) { Header = br.ReadBytes(GFPackHeader.SIZE).ToClass(); Debug.Assert(Header.MAGIC == Magic); + Pointers = new GFPackPointers(br, Header.CountFolders); - HashPaths = new FileHashPath[Header.CountFiles]; - for (int i = 0; i < HashPaths.Length; i++) - HashPaths[i] = br.ReadBytes(FileHashPath.SIZE).ToClass(); + Debug.Assert(Pointers.PtrHashPaths == br.BaseStream.Position); + HashAbsolute = new FileHashAbsolute[Header.CountFiles]; + for (int i = 0; i < HashAbsolute.Length; i++) + HashAbsolute[i] = br.ReadBytes(FileHashAbsolute.SIZE).ToClass(); - HashIndexes = new FileHashIndex[Header.CountFiles]; - for (int i = 0; i < HashIndexes.Length; i++) - HashIndexes[i] = br.ReadBytes(FileHashIndex.SIZE).ToClass(); + HashInFolder = new FileHashFolder[Header.CountFolders]; + for (int f = 0; f < HashInFolder.Length; f++) + { + Debug.Assert(Pointers.PtrHashFolders[f] == br.BaseStream.Position); + var table = HashInFolder[f] = new FileHashFolder + { + Folder = br.ReadBytes(FileHashIndex.SIZE).ToClass() + }; + table.Files = new FileHashIndex[table.Folder.FileCount]; + for (int i = 0; i < table.Files.Length; i++) + table.Files[i] = br.ReadBytes(FileHashIndex.SIZE).ToClass(); + } + Debug.Assert(Pointers.PtrFileTable == br.BaseStream.Position); FileTable = new FileData[Header.CountFiles]; for (int i = 0; i < FileTable.Length; i++) FileTable[i] = br.ReadBytes(FileData.SIZE).ToClass(); @@ -73,7 +87,7 @@ private void ReadPack(BinaryReader br) [StructLayout(LayoutKind.Sequential)] public class GFPackHeader { - public const int SIZE = 0x40; + public const int SIZE = 0x18; public ulong MAGIC = Magic; public uint Version = 0x1000; public uint IsRelocated; // bit0 @@ -87,28 +101,47 @@ public class GFPackHeader /// Count of Folders packed into the binary. /// public int CountFolders; + } - /// - /// Data offset for the table. - /// - public long PtrHashPaths; - - /// - /// Data offset for the table. - /// - public long PtrHashIndexes; - + public class GFPackPointers + { /// /// Data offset for the table. /// - public long PtrFileTable; + public long PtrFileTable; // array stored at end - public ulong Reserved1; - public ulong Reserved2; + /// + /// Data offset for the table. + /// + public long PtrHashPaths; // array stored first + + /// + /// Data offset for the table, which has a leading . + /// + public long[] PtrHashFolders; // array stored in middle + + // immediately after the pointers are the arrays + + public GFPackPointers(BinaryReader br, int folderCount) + { + PtrFileTable = br.ReadInt64(); + PtrHashPaths = br.ReadInt64(); + PtrHashFolders = new long[folderCount]; + for (int i = 0; i < PtrHashFolders.Length; i++) + PtrHashFolders[i] = br.ReadInt64(); + } + + public void Write(BinaryWriter bw) + { + bw.Write(PtrFileTable); + bw.Write(PtrHashPaths); + foreach (var table in PtrHashFolders) + bw.Write(table); + } } [StructLayout(LayoutKind.Sequential)] - public class FileHashPath + public class FileHashAbsolute { public const int SIZE = 0x08; @@ -120,6 +153,29 @@ public class FileHashPath public bool IsMatch(string fileName) => FnvHash.HashFnv1a_64(fileName) == HashFnv1aPathFull; } + public class FileHashFolder + { + public FileHashFolderInfo Folder; + public FileHashIndex[] Files; + public int GetIndexFileName(ulong hash) => Array.FindIndex(Files, z => z.HashFnv1aPathFileName == hash); + public int GetIndexFileName(string name) => Array.FindIndex(Files, z => z.IsMatch(name)); + } + + [StructLayout(LayoutKind.Sequential)] + public class FileHashFolderInfo + { + public const int SIZE = 0x10; + + /// + /// Filename (without directory details) hash. + /// + public ulong HashFnv1aPathFolderName; + public int FileCount; + public uint Padding = 0xCC; + + public bool IsMatch(string fileName) => FnvHash.HashFnv1a_64(fileName) == HashFnv1aPathFolderName; + } + [StructLayout(LayoutKind.Sequential)] public class FileHashIndex { @@ -170,31 +226,71 @@ public enum CompressionType : ushort } } - public int GetIndexFull(ulong hash) => Array.FindIndex(HashPaths, z => z.HashFnv1aPathFull == hash); - public int GetIndexFileName(ulong hash) => Array.FindIndex(HashIndexes, z => z.HashFnv1aPathFileName == hash); - public int GetIndexFileName(string name) => Array.FindIndex(HashIndexes, z => z.IsMatch(name)); + public int GetIndexFull(ulong hash) => Array.FindIndex(HashAbsolute, z => z.HashFnv1aPathFull == hash); + + public int GetIndexFileName(ulong hash) + { + foreach (var f in HashInFolder) + { + int index = f.GetIndexFileName(hash); + if (index >= 0) + return index; + } + return -1; + } + + public int GetIndexFileName(string name) + { + foreach (var f in HashInFolder) + { + int index = f.GetIndexFileName(name); + if (index >= 0) + return index; + } + return -1; + } public void LoadFiles(string[] directories, string parent, CompressionType type = CompressionType.Lz4) { - var files = directories.SelectMany(Directory.GetFiles).ToArray(); + var groups = directories.Select(Directory.GetFiles).ToArray(); + var files = groups.SelectMany(z => z).ToArray(); Header = new GFPackHeader { CountFolders = directories.Length, CountFiles = files.Length }; - HashPaths = new FileHashPath[files.Length]; - HashIndexes = new FileHashIndex[files.Length]; + HashAbsolute = new FileHashAbsolute[files.Length]; + HashInFolder = new FileHashFolder[groups.Length]; FileTable = new FileData[files.Length]; DecompressedFiles = new byte[files.Length][]; CompressedFiles = new byte[files.Length][]; + + for (int f = 0; f < groups.Length; f++) + { + var folderFiles = groups[f]; + var folderName = Path.GetDirectoryName(directories[f]); + var table = HashInFolder[f] = new FileHashFolder(); + table.Folder = new FileHashFolderInfo + { + FileCount = folderFiles.Length, + HashFnv1aPathFolderName = FnvHash.HashFnv1a_64(folderName), + }; + table.Files = new FileHashIndex[folderFiles.Length]; + for (int i = 0; i < folderFiles.Length; i++) + { + var file = folderFiles[i]; + int index = Array.IndexOf(files, folderFiles[i]); + var nameshort = Path.GetFileName(file); + ulong hashShort = FnvHash.HashFnv1a_64(nameshort); + table.Files[i] = new FileHashIndex { HashFnv1aPathFileName = hashShort, Index = index }; + } + } + for (var i = 0; i < files.Length; i++) { var file = files[i]; - var nameshort = Path.GetFileName(file); var namelong = file.Substring(file.IndexOf(parent, StringComparison.Ordinal)); - ulong hashShort = FnvHash.HashFnv1a_64(nameshort); ulong hashFull = FnvHash.HashFnv1a_64(namelong); - HashPaths[i] = new FileHashPath {HashFnv1aPathFull = hashFull}; - HashIndexes[i] = new FileHashIndex {HashFnv1aPathFileName = hashShort, Index = i}; - FileTable[i] = new FileData {Type = type}; + HashAbsolute[i] = new FileHashAbsolute { HashFnv1aPathFull = hashFull }; + FileTable[i] = new FileData { Type = type }; DecompressedFiles[i] = FileMitm.ReadAllBytes(file); } Modified = true; @@ -219,7 +315,8 @@ public byte[] Write() entry.OffsetPacked = (int)bw.BaseStream.Position; bw.Write(c); - // no padding + while (bw.BaseStream.Position % 0x10 != 0) // pad to nearest 0x10 alignment + bw.Write((byte)0); } bw.BaseStream.Position = 0; WriteHeaderTableList(bw); @@ -263,13 +360,23 @@ public void CancelEdits() private void WriteHeaderTableList(BinaryWriter bw) { bw.Write(Header.ToBytesClass()); - Header.PtrHashPaths = bw.BaseStream.Position; - foreach (var hp in HashPaths) + Pointers.Write(bw); + Pointers.PtrHashPaths = bw.BaseStream.Position; + foreach (var hp in HashAbsolute) bw.Write(hp.ToBytesClass()); - Header.PtrHashIndexes = bw.BaseStream.Position; - foreach (var hi in HashIndexes) - bw.Write(hi.ToBytesClass()); - Header.PtrFileTable = bw.BaseStream.Position; + + for (var f = 0; f < Pointers.PtrHashFolders.Length; f++) + { + Pointers.PtrHashFolders[f] = bw.BaseStream.Position; + + var folder = HashInFolder[f]; + folder.Folder.FileCount = folder.Files.Length; + bw.Write(folder.Folder.ToBytesClass()); + foreach (var hi in folder.Files) + bw.Write(hi.ToBytesClass()); + } + + Pointers.PtrFileTable = bw.BaseStream.Position; foreach (var ft in FileTable) bw.Write(ft.ToBytesClass()); } @@ -285,14 +392,22 @@ public void Dump(string path, ContainerHandler handler) { handler.Initialize(FileTable.Length); string format = this.GetFileFormatString(); - for (var i = 0; i < FileTable.Length; i++) + + foreach (var grp in HashInFolder) { - var hashFull = HashPaths[i].HashFnv1aPathFull; - var fn = $"{i.ToString(format)} {hashFull:X16}.bin"; - var loc = Path.Combine(path ?? FilePath, fn); - var data = DecompressedFiles[i]; - FileMitm.WriteAllBytes(loc, data); - handler.StepFile(i, fileName: fn); + var dirName = grp.Folder.HashFnv1aPathFolderName; + foreach (var f in grp.Files) + { + var index = f.Index; + var name = f.HashFnv1aPathFileName; + + var fn = $"{index.ToString(format)} {name:X16}.bin"; + var data = DecompressedFiles[f.Index]; + + var subfolder = HashInFolder.Length == 1 ? fn : Path.Combine(dirName.ToString("X16"), fn); + var loc = Path.Combine(path ?? FilePath, subfolder); + FileMitm.WriteAllBytes(loc, data); + } } } }