diff --git a/include/FileContainerReader.h b/include/FileContainerReader.h new file mode 100644 index 0000000..94b4984 --- /dev/null +++ b/include/FileContainerReader.h @@ -0,0 +1,94 @@ +#ifndef _FILECONTAINERREADER_H +#define _FILECONTAINERREADER_H + +#include "typeDefs.h" + +#define FILE_NAME_LENGTH 16 + +/** + * @brief This class provides functionality to read files from the file container format + * created by the make-file-container tool. + * + * This format consists of multiple compressed chunks. + * + * It is done this way to keep the decompression buffer small (because we can't afford to consume too much IWRAM) + * but still allow for large file containers and bundle files together. + * The chunk size can be configured when creating the file container, and the + * FileContainerReader will read and decompress chunks on demand when seeking or reading data. + */ +class FileContainerReader +{ +public: + FileContainerReader(const u8 **chunkList, u32 chunkCount, u32 chunkSize = 4096); + ~FileContainerReader(); + + /** + * @brief This function initializes the FileContainerReader. It should be called before any other function is used. + * The reason for this initialization step is to give you control over the moment when the first chunk is decompressed. + * This can be useful if you want to perform some setup or checks before the decompression process begins. By calling init() explicitly, you can ensure that the FileContainerReader is ready to handle file reading operations at the appropriate time in your application. + * + * Note: because of this, we also only pass the decompression buffer here + * If the decompression buffer is not large enough to hold the chunks, + * the function will return false and the FileContainerReader will not be initialized. In this case, you should provide a larger decompression buffer and call init() again. + * + * If it DOES return true, please note that the FileContainerReader will keep using the decompressionBuffer_ until + * it gets destroyed. + */ + bool init(u8 *decompressionBuffer, u32 decompressionBufferSize); + + /** + * @brief IF the file container stores the file names, + * this function returns a pointer to the file name of the file at the given index. + * Otherwise, it returns nullptr. + */ + const char* getFileName(u32 fileIndex); + + /** + * @brief This function returns the size of the file at the given index. + */ + u32 getFileSize(u32 fileIndex) const; + + /** + * @brief This function seeks to the beginning of the file at the given index. + */ + void seekToFile(u32 fileIndex); + + /** + * @brief Read data from the current position in the file container into the provided buffer. + */ + void read(u8 *buffer, u32 size); +protected: +private: + /** + * @brief This function calculates the file offset for the specified file index + * The file offset is determined by reading the file index (which contains the file entry sizes) + * and summing up the sizes of all previous entries + header size + optional name table size + * (if names are stored). + */ + u32 getFileOffset(u32 fileIndex) const; + + /** + * @brief This function seeks to the specified absolute offset in the file container data. + * Absolute offset refers to an offset across the file container chunks. + */ + void seek(u32 offset); + + /** + * @brief This function just uncompresses the chunk at the specified index + * into decompressionBuffer_. + */ + void uncompressChunk(u32 chunkIndex); + + char fileNameBuffer_[FILE_NAME_LENGTH + 1]; + u8 *decompressionBuffer_; + const u8 **chunkList_; + u32 chunkCount_; + u32 chunkSize_; + u16 *fileIndex_; + u32 curChunkIndex_; + u32 curPos_; + u32 fileCount_; + bool hasNames_; +}; + +#endif \ No newline at end of file diff --git a/source/FileContainerReader.cpp b/source/FileContainerReader.cpp new file mode 100644 index 0000000..e43b816 --- /dev/null +++ b/source/FileContainerReader.cpp @@ -0,0 +1,154 @@ +#include "FileContainerReader.h" +#include + +#include + +#define FILE_CONTAINER_HEADER_SIZE 4 +#define FILE_INDEX_ENTRY_SIZE 2 + +FileContainerReader::FileContainerReader(const u8 **chunkList, u32 chunkCount, u32 chunkSize) + : fileNameBuffer_() + , decompressionBuffer_(nullptr) + , chunkList_(chunkList) + , chunkCount_(chunkCount) + , chunkSize_(chunkSize) + , fileIndex_(nullptr) + , curChunkIndex_(0) + , curPos_(0) + , fileCount_(0) + , hasNames_(false) +{ +} + +FileContainerReader::~FileContainerReader() +{ + if (fileIndex_) + { + delete[] fileIndex_; + fileIndex_ = nullptr; + } +} + +bool FileContainerReader::init(u8 *decompressionBuffer, u32 decompressionBufferSize) +{ + if(decompressionBufferSize < chunkSize_) + { + return false; + } + + decompressionBuffer_ = decompressionBuffer; + + // decompress the first chunk to get the header and file index information. + uncompressChunk(0); + + fileCount_ = decompressionBuffer_[0]; + hasNames_ = decompressionBuffer_[1]; + + // The header size is 4 bytes. + curPos_ = FILE_CONTAINER_HEADER_SIZE; + + // The file index just stores the file sizes for each file entry. + // We can calculate the file offset based on them. + const u32 indexSize = fileCount_ * FILE_INDEX_ENTRY_SIZE; + fileIndex_ = new u16[fileCount_]; + read((u8*)fileIndex_, indexSize); + + return true; +} + +const char* FileContainerReader::getFileName(u32 fileIndex) +{ + if(!hasNames_) + { + return nullptr; + } + + // the file names are stored right after the file index, and each name is 16 bytes long. + const u32 indexSize = fileCount_ * FILE_INDEX_ENTRY_SIZE; + const u32 nameTableStartPos = FILE_CONTAINER_HEADER_SIZE + indexSize; + seek(nameTableStartPos + (fileIndex * FILE_NAME_LENGTH)); + + // Now copy the file name to a local buffer. + // the reason is that not every string is necessarily null terminated in the fileocontainer data. + read((u8*)fileNameBuffer_, FILE_NAME_LENGTH); + fileNameBuffer_[FILE_NAME_LENGTH] = '\0'; + return fileNameBuffer_; +} + +u32 FileContainerReader::getFileSize(u32 index) const +{ + // fileIndex_ stores the file sizes for each entry. + return *(fileIndex_ + index); +} + +void FileContainerReader::seekToFile(u32 fileIndex) +{ + const u32 fileOffset = getFileOffset(fileIndex); + + seek(fileOffset); +} + +void FileContainerReader::read(u8 *buffer, u32 size) +{ + u32 bytesRemaining = size; + + while(bytesRemaining > 0) + { + if(curPos_ == chunkSize_) + { + // we need to decompress a different chunk + ++curChunkIndex_; + uncompressChunk(curChunkIndex_); + } + + const u32 bytesRemainingInChunk = chunkSize_ - curPos_; + const u32 bytesToRead = bytesRemaining < bytesRemainingInChunk ? bytesRemaining : bytesRemainingInChunk; + + memcpy(buffer, decompressionBuffer_ + curPos_, bytesToRead); + bytesRemaining -= bytesToRead; + buffer += bytesToRead; + curPos_ += bytesToRead; + } +} + +u32 FileContainerReader::getFileOffset(u32 entryIndex) const +{ + // files data start after: + // - header (4 bytes) + // - file index (fileCount * 2 bytes) + // - optional name table (fileCount * 16 bytes) + // + u32 offset = FILE_CONTAINER_HEADER_SIZE + (fileCount_ * FILE_INDEX_ENTRY_SIZE); + if(hasNames_) + { + offset += fileCount_ * FILE_NAME_LENGTH; + } + + // accumulate the sizes of all previous entries to get the file offset + for(u32 i = 0; i < entryIndex; ++i) + { + offset += getFileSize(i); + } + + return offset; +} + +void FileContainerReader::seek(u32 offset) +{ + const u32 targetChunkIndex = offset / chunkSize_; + + if(curChunkIndex_ != targetChunkIndex) + { + // we need to decompress a different chunk + uncompressChunk(targetChunkIndex); + } + + curPos_ = offset % chunkSize_; +} + +void FileContainerReader::uncompressChunk(u32 chunkIndex) +{ + LZ77UnCompWram(chunkList_[chunkIndex], decompressionBuffer_); + curChunkIndex_ = chunkIndex; + curPos_ = 0; +} \ No newline at end of file diff --git a/tools/make-file-container/src/main.cpp b/tools/make-file-container/src/main.cpp index a5023bc..da17d64 100644 --- a/tools/make-file-container/src/main.cpp +++ b/tools/make-file-container/src/main.cpp @@ -17,7 +17,6 @@ typedef struct { - uint16_t offset; uint16_t size; char altName[FILE_RECORD_NAME_LENGTH]; char path[PATH_BUFFER_SIZE]; @@ -209,24 +208,19 @@ static void writeHeader(FileContainerChunkWriter &writer, const ContainerMetadat writer.writeUint16(meta.chunkSize); } +/** + * @brief This function writes the file index. + * It just stores the file sizes for every entry, not the offsets. + * The reason is two-fold: + * + * - It makes it much better for compression, as file sizes will repeat more often than accumulating offsets + * - If you'd use offsets and base your file size calculation on them, you'd get into trouble with the last entry. + */ static void writeIndex(FileContainerChunkWriter &writer, ContainerMetadata &meta) { - // first we need to determine where the actual file data starts. - // That is, after the header, index entries and optional names. - constexpr uint16_t headerSize = 4; - uint16_t current_offset = headerSize + sizeof(uint16_t) * meta.entries.size(); - - if(meta.hasNames) - { - current_offset += FILE_RECORD_NAME_LENGTH * meta.entries.size(); - } - for(auto &entry : meta.entries) { - entry.offset = current_offset; - current_offset += entry.size; - - writer.writeUint16(entry.offset); + writer.writeUint16(entry.size); } }