From 7d26f68bec26c6abc63dc534b9ee4f956bcdd653 Mon Sep 17 00:00:00 2001 From: Rachel Date: Sun, 10 May 2026 14:52:22 -0700 Subject: [PATCH] tools: Deprecate csv2bin --- tools/csv2bin/CsvFile.cpp | 252 -------------------------------- tools/csv2bin/CsvFile.h | 69 --------- tools/csv2bin/Manifest.cpp | 265 ---------------------------------- tools/csv2bin/Manifest.h | 289 ------------------------------------- tools/csv2bin/Options.cpp | 273 ----------------------------------- tools/csv2bin/Options.h | 45 ------ tools/csv2bin/global.h | 22 --- tools/csv2bin/main.cpp | 21 --- tools/csv2bin/meson.build | 13 -- tools/meson.build | 1 - 10 files changed, 1250 deletions(-) delete mode 100644 tools/csv2bin/CsvFile.cpp delete mode 100644 tools/csv2bin/CsvFile.h delete mode 100644 tools/csv2bin/Manifest.cpp delete mode 100644 tools/csv2bin/Manifest.h delete mode 100644 tools/csv2bin/Options.cpp delete mode 100644 tools/csv2bin/Options.h delete mode 100644 tools/csv2bin/global.h delete mode 100644 tools/csv2bin/main.cpp delete mode 100644 tools/csv2bin/meson.build diff --git a/tools/csv2bin/CsvFile.cpp b/tools/csv2bin/CsvFile.cpp deleted file mode 100644 index 636532834c..0000000000 --- a/tools/csv2bin/CsvFile.cpp +++ /dev/null @@ -1,252 +0,0 @@ -#include -#include "CsvFile.h" - -void CsvFile::ParseRow(std::string &line, std::vector &row, bool resize) { - std::string entry, qbuf; - bool isQuoted = false; - int i = 0; - if (resize) { - row.clear(); - } - while (line[0] == '\r') { - line = line.substr(1); - } - while (line[line.size() - 1] == '\r') { - line = line.substr(0, line.size() - 1); - } - std::stringstream line_s(line); - while (std::getline(line_s, entry, ',')) { - if (!isQuoted && entry[0] == '"') { - isQuoted = true; - entry = entry.substr(1); - } - if (isQuoted) { - if (entry[entry.size() - 1] == '"') { - isQuoted = false; - entry = entry.substr(0, entry.size() - 1); - } - qbuf += entry; - if (!isQuoted) { - if (resize) { - row.push_back(qbuf); - } else { - row[i++] = qbuf; - } - qbuf.clear(); - } - } else { - if (resize) { - row.push_back(entry); - } else { - row[i++] = entry; - } - } - } - assert(!isQuoted); -} - -void CsvFile::FromFile(const fs::path &filename, bool has_header) { - std::ifstream handle(filename); - std::string line; - std::stringstream filebuf; - - // Read the whole file at once - filebuf << handle.rdbuf(); - - // Read the first row - line = filebuf.str().substr(0, filebuf.str().find_first_of("\r\n")); - - // Calculate the number of rows - size_t pos = 0; - for ( - _nrow = !has_header; - pos != std::string::npos && - (pos = filebuf.str().find_first_of("\r\n", pos), pos != std::string::npos); - _nrow++ - ) { - pos = filebuf.str().find_first_not_of("\r\n", pos); - if (pos == std::string::npos) { - break; - } - } - - // Calculate the number of columns - pos = 0; - for ( - _ncol = 1; - pos = line.find(',', pos), pos++ != std::string::npos; - _ncol++ - ) {} - - // Preallocate the rows and colnames - _rows.resize(_nrow); - _colnames.resize(_ncol); - - // Parse the header, or set a dummy header - if (has_header) { - ParseRow(line, _colnames, false); - pos = filebuf.str().find_first_of("\r\n", pos); - if (pos != std::string::npos) { - pos = filebuf.str().find_first_not_of("\r\n", pos); - } - } else { - int i = 1; - for (std::string & name : _colnames) { - name = "V" + std::to_string(i++); - } - pos = 0; - } - - // Parse the rows - size_t last_pos = pos; - for ( - auto row = _rows.begin(); - row != _rows.end() && - (last_pos = pos, last_pos != std::string::npos) && - (pos = filebuf.str().find_first_of("\r\n", pos), pos != std::string::npos); - row++ - ) { - line = filebuf.str().substr(last_pos, pos - last_pos); - row->resize(_ncol); - ParseRow(line, *row, false); - pos = filebuf.str().find_first_not_of("\r\n", pos); - if (pos == std::string::npos) { - break; - } - } -} - -bool CsvFile::GetRow(size_t i, std::vector &row) const { - if (i >= _nrow) { - row.clear(); - return false; - } else { - row = _rows[i]; - return true; - } -} - -bool CsvFile::GetCol(size_t i, std::vector &col) const { - if (i >= _ncol) { - col.clear(); - return false; - } else { - int j = 0; - for (const auto & row : _rows) { - col[j++] = row[i]; - } - return true; - } -} - -bool CsvFile::GetCol(const std::string &name, std::vector &col) const { - auto it = std::find(_colnames.cbegin(), _colnames.cend(), name); - if (it != _colnames.cend()) { - size_t i = it - _colnames.cbegin(); - return GetCol(i, col); - } else { - col.clear(); - return false; - } -} - -bool CsvFile::GetCell(size_t row, size_t col, std::string &cell) const { - if (row >= _nrow || col >= _ncol) { - cell.clear(); - return false; - } else { - cell = _rows[row][col]; - return true; - } -} - -bool CsvFile::GetCell(const std::string &colname, size_t row, std::string &cell) const { - auto it = std::find(_colnames.cbegin(), _colnames.cend(), colname); - if (it != _colnames.cend()) { - size_t i = it - _colnames.cbegin(); - return GetCell(row, i, cell); - } else { - cell.clear(); - return false; - } -} - -bool CsvFile::SetRow(size_t i, const std::vector &row) { - if (i >= _nrow) { - return false; - } else { - _rows[i] = row; - return true; - } -} - -bool CsvFile::SetCol(size_t i, const std::vector &col) { - if (i >= _ncol) { - return false; - } else { - int j = 0; - for (auto & row : _rows) { - row[i] = col[j++]; - } - return true; - } -} - -bool CsvFile::SetCol(const std::string &name, const std::vector &col) { - auto it = std::find(_colnames.cbegin(), _colnames.cend(), name); - if (it != _colnames.cend()) { - size_t i = it - _colnames.cbegin(); - return SetCol(i, col); - } else { - return false; - } -} - -bool CsvFile::SetCell(size_t row, size_t col, const std::string &cell) { - if (row >= _nrow || col >= _ncol) { - return false; - } else { - _rows[row][col] = cell; - return true; - } -} - -bool CsvFile::SetCell(const std::string &colname, size_t row, const std::string &cell) { - auto it = std::find(_colnames.cbegin(), _colnames.cend(), colname); - if (it != _colnames.cend()) { - size_t i = it - _colnames.cbegin(); - return SetCell(row, i, cell); - } else { - return false; - } -} - -void CsvFile::WriteRow(std::ofstream &ofile, const std::vector &row) { - bool first = true; - for (const auto & entry : row) { - if (!first) { - ofile << ','; - } else { - first = false; - } - bool has_comma = (entry.find(',') != std::string::npos); - if (has_comma) { - ofile << '"'; - } - ofile << entry; - if (has_comma) { - ofile << '"'; - } - } - ofile << '\n'; -} - -void CsvFile::ToFile(const fs::path &filename, bool write_header) { - std::ofstream ofile(filename); - if (write_header) { - WriteRow(ofile, _colnames); - } - for (auto const & row : _rows) { - WriteRow(ofile, row); - } -} diff --git a/tools/csv2bin/CsvFile.h b/tools/csv2bin/CsvFile.h deleted file mode 100644 index 60938915ca..0000000000 --- a/tools/csv2bin/CsvFile.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef GUARD_CSVFILE_H -#define GUARD_CSVFILE_H - -#include "global.h" -#include - -class CsvFile { - std::vector _colnames; - std::vector> _rows; - size_t _nrow = 0; - size_t _ncol = 0; - bool _has_header = true; - static void ParseRow(std::string &line, std::vector &row, bool resize = true); - static void WriteRow(std::ofstream &ofile, std::vector const &row); -public: - CsvFile() = default; - explicit CsvFile(const fs::path & filename, bool has_header = true) { FromFile(filename, has_header); }; - void FromFile(const fs::path & filename, bool has_header = true); - - void ToFile(const fs::path & filename, bool write_header = true); - bool GetRow(size_t i, std::vector &row) const; - bool GetCol(size_t i, std::vector &col) const; - bool GetCol(const std::string &name, std::vector &col) const; - bool GetCell(size_t row, size_t col, std::string &cell) const; - bool GetCell(const std::string &colname, size_t row, std::string &cell) const; - bool SetRow(size_t i, const std::vector &row); - bool SetCol(size_t i, const std::vector &col); - bool SetCol(const std::string &name, const std::vector &col); - bool SetCell(size_t row, size_t col, const std::string &cell); - bool SetCell(const std::string &colname, size_t row, const std::string &cell); - std::vector>::iterator begin() { return _rows.begin(); } - std::vector>::iterator end() { return _rows.end(); } - std::vector>::const_iterator cbegin() { return _rows.cbegin(); } - std::vector>::const_iterator cend() { return _rows.cend(); } - std::vector>::reverse_iterator rbegin() { return _rows.rbegin(); } - std::vector>::reverse_iterator rend() { return _rows.rend(); } - std::vector>::const_reverse_iterator crbegin() { return _rows.crbegin(); } - std::vector>::const_reverse_iterator crend() { return _rows.crend(); } - [[nodiscard]] size_t nrow() const { return _nrow; } - [[nodiscard]] size_t ncol() const { return _ncol; } - std::vector &operator[](size_t i) { return _rows[i]; } - void resize_internal() { - _rows.resize(_nrow); - for (auto & row : _rows) { - row.resize(_ncol); - } - } - void resize(size_t nrow, size_t ncol) { - if (_nrow != nrow || _ncol != ncol) { - _nrow = nrow; - _ncol = ncol; - resize_internal(); - } - } - template - void SetColnames(Iter _begin, Iter _end) { - _colnames.assign(_begin, _end); - if (_ncol != _colnames.size()) { - _ncol = _colnames.size(); - resize_internal(); - } - } - std::vector &GetColnames() { - return _colnames; - } -}; - - -#endif //GUARD_CSVFILE_H diff --git a/tools/csv2bin/Manifest.cpp b/tools/csv2bin/Manifest.cpp deleted file mode 100644 index e82de6ff61..0000000000 --- a/tools/csv2bin/Manifest.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "Manifest.h" - -std::map, std::map> HeaderCache { - {{"bool", ""}, { - {"false", 0}, - {"true", 1} - }} -}; - -void ColumnSpec::translate_width(std::string &width, int &bytes, int &bits) { - int dotpos = width.find('.'); - bytes = std::stoi(width.substr(0, dotpos)); - if (dotpos == std::string::npos) { - bits = 0; - } else { - bits = std::stoi(width.substr(dotpos + 1)); - } -} - -void ColumnSpec::_init(int _width, const fs::path &headerfile, const std::string &prefix, int _nbits) { - width = _width; - nbits = _nbits; - if (!headerfile.empty()) { - try { - constants = HeaderCache.at({headerfile, prefix}); - } catch (const std::out_of_range &e) { - std::ifstream handle(headerfile); - std::regex pattern("#define +(" + prefix + "\\w+) +(\\d+)$"); - std::string line; - std::smatch results; - while (std::getline(handle, line)) { - if (std::regex_match(line, results, pattern)) { - constants[results[1]] = std::stoi(results[2]); - } - } - HeaderCache[{headerfile, prefix}] = constants; - } - } -} - -ColumnSpec::ColumnSpec(int _width, const fs::path& headerfile, const std::string &prefix, int _nbits) { - _init(_width, headerfile, prefix, _nbits); -} - -ColumnSpec::ColumnSpec(std::string &_width, const fs::path &headerfile, const std::string &prefix) { - int sign; - int nbit; - int nbytes; - - if (_width == "skip") { - _init(skip, headerfile, prefix); - return; - } - - if (_width.substr(0, 3) == "pad") { - std::string __width = _width.substr(3); - translate_width(__width, nbytes, nbit); - _init(pad | nbytes, headerfile, prefix, nbit); - return; - } - - switch (_width[0]) { - case 's': - sign = -1; - break; - case 'u': - sign = 1; - break; - default: - throw std::invalid_argument("width param must be a valid fixed-width type spec"); - } - - std::string __width = _width.substr(1); - translate_width(__width, nbytes, nbit); - switch (nbytes) { - case 8: - case 16: - case 32: - case 64: - sign *= nbytes / 8; - break; - default: - throw std::invalid_argument("width param must be a valid fixed-width type spec"); - } - - _init(sign, headerfile, prefix, nbit); -} - -Manifest::Manifest(const fs::path &filename, std::vector &header_dirs) { - read(filename, header_dirs); -} - -void Manifest::read(const fs::path &filename, std::vector &header_dirs) { - std::ifstream strm(filename); - std::string line; - while (std::getline(strm, line)) { - line = line.substr(0, line.find_last_not_of(" \t\r\n") + 1); - fs::path headerfile; - std::string prefix; - - // Tokenize with colons - std::vector tokens; - size_t s = 0; - size_t e; - do { - e = line.find(':', s); - tokens.emplace_back(line.substr(s, e - s)); - s = e + 1; - } while (e != std::string::npos); - - std::string name = tokens.at(0); - try { - mapping.at(name); - throw std::invalid_argument("duplicate column in manifest: " + name); - } catch (const std::out_of_range &e) { - // discard silently, we gucci - } - std::string width = tokens.at(1); - if (tokens.size() > 2) { - if (tokens.size() > 3) { - prefix = tokens[3]; - } - fs::path header_name = tokens[2]; - if (header_name == "bool") { - headerfile = "bool"; - } else { - for (auto &root: header_dirs) { - if (fs::exists(root / header_name)) { - headerfile = root / header_name; - break; - } - } - } - } - mapping[name] = ColumnSpec(width, headerfile, prefix); - colnames.emplace_back(name); - } -} - -ColumnSpec &Manifest::operator[](const std::string &name) { - return mapping[name]; -} - -size_t Manifest::size(const int alignment) const { - size_t ret = 0; - size_t bitpos = 0; - for (const auto & name : colnames) { - auto &spec = mapping.at(name); - if (spec.is_skipped()) { - continue; - } - size_t bytect = spec.size(); - size_t aln = spec.get_alignment(); - size_t bitct = spec.num_bits(); - if (bitpos != 0 && bitct == 0) { - bitpos = 0; - ret++; - } - if (bitpos == 0 && aln != 1) { - ret += aln - 1; - ret &= ~(aln - 1); - } - if (bitct != 0) { - bitpos += bitct; - if (bitpos >= 8 * bytect) { - ret += bytect; - bitpos -= 8 * bytect; - } - } else { - ret += bytect; - } - } - // Word align - if (alignment != 0) { - ret += alignment - 1; - ret &= ~(alignment - 1); - } - return ret; -} - -BufferedRowConverter::BufferedRowConverter(Manifest &_manifest, CsvFile &_csvFile, unsigned char _padval): - manifest(_manifest), - csvFile(_csvFile), - padval(_padval) -{ - buffer.resize(manifest.size()); - carriage_return(); - byte_cursor = 0; - bit_cursor = 0; - row_cursor = 0; -} - -std::ifstream &operator>>(std::ifstream &strm, BufferedRowConverter &cvtr) { - std::ios::iostate state = strm.rdstate(); - size_t pos = strm.tellg(); - strm.read((char *)cvtr.buffer.data(), cvtr.buffer.size()); - cvtr.to_strings(); - cvtr++; - return strm; -} - -std::ofstream &operator<<(std::ofstream &strm, BufferedRowConverter &cvtr) { - cvtr.to_bytes(); - strm.write((char *)cvtr.buffer.data(), cvtr.buffer.size()); - cvtr++; - return strm; -} - -void BufferedRowConverter::to_strings() { - if (row_cursor >= csvFile.nrow()) { - throw std::out_of_range("invalid row idx"); - } - std::vector &row = csvFile[row_cursor]; - size_t column_i = 0; - for (const auto colname : manifest.colnames) { - const ColumnSpec &spec = manifest[colname]; - if (spec.is_skipped()) { - row.at(column_i++) = spec[row_cursor]; - continue; - } else { - align(spec.size(), spec.num_bits()); - unsigned long long val = get(spec.type(), spec.num_bits()); - if (spec.is_padding()) { - if (val != 0) { - std::cerr << "csv2bin warning: nonzero data in padding field may result in data loss" << std::endl; - } - } else { - row.at(column_i++) = spec[val]; - } - advance(spec.size(), spec.num_bits()); - } - } -} - -void BufferedRowConverter::to_bytes() { - if (row_cursor >= csvFile.nrow()) { - throw std::out_of_range("invalid row idx"); - } - std::vector &row = csvFile[row_cursor]; - size_t column_i = 0; - for (const auto colname : manifest.colnames) { - const ColumnSpec &spec = manifest[colname]; - if (spec.is_skipped()) { - column_i++; - continue; - } else { - align(spec.get_alignment(), spec.num_bits()); - unsigned long long val; - if (spec.is_padding()) { - val = 0; - } else { - val = spec[row.at(column_i++)]; - } - set(val, spec.type(), spec.num_bits()); - advance(spec.size(), spec.num_bits()); - } - } - if (bit_cursor != 0) { - const ColumnSpec &spec = manifest[*manifest.colnames.crbegin()]; - byte_cursor += spec.size(); - } - while (byte_cursor < buffer.size()) { - buffer[byte_cursor++] = padval; - } -} diff --git a/tools/csv2bin/Manifest.h b/tools/csv2bin/Manifest.h deleted file mode 100644 index e89b41c92b..0000000000 --- a/tools/csv2bin/Manifest.h +++ /dev/null @@ -1,289 +0,0 @@ -#ifndef GUARD_MANIFEST_H -#define GUARD_MANIFEST_H - -#include "global.h" -#include "CsvFile.h" - -template -C* to_array(C* buf, const T val, off_t offset = 0) { - for (int i = 0; i < sizeof(T); i++) { - buf[offset + i] = (val >> (8 * i * sizeof(C))); - } - return buf; -} - -template -T from_array(const C* buf, off_t offset = 0) { - T ret = 0; - for (int i = 0; i < sizeof(T); i++) { - ret |= buf[offset + i] << (8 * i * sizeof(C)); - } - return ret; -} - -template -std::vector & to_array(std::vector & buf, const T val, off_t offset = 0) { - for (int i = 0; i < sizeof(T); i++) { - buf[offset + i] = (val >> (8 * i * sizeof(C))); - } - return buf; -} - -template -T from_array(const std::vector & buf, off_t offset = 0) { - T ret = 0; - for (int i = 0; i < sizeof(T); i++) { - ret |= buf[offset + i] << (8 * i * sizeof(C)); - } - return ret; -} - -class padding_warning : public std::exception { - std::string _what; -public: - padding_warning(const char *c) : _what(c) {} - padding_warning(const std::string &s) : _what(s) {} - const char * what() const noexcept { return _what.c_str(); } -}; - -class ColumnSpec { - typedef int width_t; - static const width_t u8 = 1; - static const width_t u16 = 2; - static const width_t u32 = 4; - static const width_t u64 = 8; - static const width_t s8 = -1; - static const width_t s16 = -2; - static const width_t s32 = -4; - static const width_t s64 = -8; - static const width_t skip = 256; - static const width_t pad = 512; - - width_t width = 0; // number of bytes. positive = unsigned, negative = signed - unsigned char nbits = 0; - std::map constants; // from a C header, specified in the manifest - - void _init(int _width, const fs::path &headerfile = "", const std::string &prefix = "", int _nbits = 0); - static void translate_width(std::string &width, int &bytes, int &bits); -public: - ColumnSpec() = default; - explicit ColumnSpec(int _width, const fs::path& headerfile = "", const std::string &prefix = "", int _nbits = 0); - explicit ColumnSpec(std::string &_width, const fs::path& headerfile = "", const std::string &prefix = ""); - size_t size() const { - if (is_skipped()) { - return 0; - } - if (is_padding()) { - return width & 0xFF; - } - return abs(width); - } - int type() const { - if (is_skipped()) { - return 0; - } - if (is_padding()) { - return width & 0xFF; - } - return width; - } - bool is_signed() const { return width < 0; } - bool is_init() const { return width != 0; } - bool is_padding() const { return width > 0 && (width & ~0xFF) == pad; } - bool is_skipped() const { return width == skip; } - bool is_bitfield() const { return nbits != 0; } - unsigned num_bits() const { return nbits; } - unsigned get_alignment() const { - if (is_skipped() || is_padding()) { - return 1; - } - return size(); - } - const std::string operator[](int i) const { - auto it = std::find_if(constants.cbegin(), constants.cend(), [&](const auto pair) { return pair.second == i; }); - if (it == constants.end()) { - return std::to_string(i); - } - return it->first; - } - int operator[](const std::string &key) const { - try { - return constants.at(key); - } catch (std::out_of_range &e) { - return std::stoi(key); - } - } -}; - -// File format: newline separated -class Manifest { - std::map mapping; // maps column name to column specification -public: - std::vector colnames; // column ordering - using manifest_iter = std::map::iterator; - using manifest_const_iter = std::map::const_iterator; - using manifest_reverse_iter = std::map::reverse_iterator; - using manifest_const_reverse_iter = std::map::const_reverse_iterator; - Manifest() = default; - explicit Manifest(fs::path const &filename, std::vector &header_dirs); - void read(fs::path const &filename, std::vector &header_dirs); - ColumnSpec &operator[](const std::string &name); - manifest_iter begin() { return mapping.begin(); } - manifest_iter end() { return mapping.end(); } - manifest_const_iter cbegin() { return mapping.cbegin(); } - manifest_const_iter cend() { return mapping.cend(); } - manifest_reverse_iter rbegin() { return mapping.rbegin(); } - manifest_reverse_iter rend() { return mapping.rend(); } - manifest_const_reverse_iter crbegin() { return mapping.crbegin(); } - manifest_const_reverse_iter crend() { return mapping.crend(); } - [[nodiscard]] size_t size(const int alignment = 4) const; - template - void assign(Iter first, Iter last) { - mapping.clear(); - for (auto pair = first; pair != last; pair ++) { - mapping[pair->first] = pair->second; - } - } -}; - -class BufferedRowConverter { - Manifest &manifest; - CsvFile &csvFile; - std::vectorbuffer; - off_t byte_cursor = 0; - off_t bit_cursor = 0; - off_t row_cursor = 0; - unsigned char padval = 0; -public: - BufferedRowConverter(Manifest &_manifest, CsvFile &_csvFile, unsigned char _padval = 0); - void to_strings(); - void to_bytes(); - friend std::ifstream &operator>>(std::ifstream &strm, BufferedRowConverter &cvtr); - friend std::ofstream &operator<<(std::ofstream &strm, BufferedRowConverter &cvtr); - BufferedRowConverter &operator++() { - if (row_cursor >= csvFile.nrow()) { - throw std::out_of_range("BufferedRowConverter++"); - } - row_cursor++; - return *this; - }; - BufferedRowConverter &operator++(int i) { - if (row_cursor + i > csvFile.nrow()) { - throw std::out_of_range("BufferedRowConverter++"); - } - row_cursor++; - carriage_return(); - return *this; - }; - unsigned bitmask(unsigned nbits) const { - return ((1ul << nbits) - 1) << bit_cursor; - } - void align(const unsigned alignment, const unsigned bitcount) { - if (bitcount == 0 && bit_cursor != 0) { - byte_cursor++; - bit_cursor = 0; - } - if (alignment == 1) return; - if (byte_cursor & (alignment - 1)) { - byte_cursor += alignment - 1; - byte_cursor &= ~(alignment - 1); - bit_cursor = 0; - } - } - unsigned long long get(int width, int numbits = 0) const { - unsigned long ret; - if (byte_cursor + abs(width) > buffer.size()) { - throw std::out_of_range("BufferedRowConverter::get"); - } - switch (width) { - case 1: - ret = from_array(buffer, byte_cursor); - break; - case 2: - ret = from_array(buffer, byte_cursor); - break; - case 4: - ret = from_array(buffer, byte_cursor); - break; - case 8: - ret = from_array(buffer, byte_cursor); - break; - case -1: - ret = from_array(buffer, byte_cursor); - break; - case -2: - ret = from_array(buffer, byte_cursor); - break; - case -4: - ret = from_array(buffer, byte_cursor); - break; - case -8: - ret = from_array(buffer, byte_cursor); - break; - default: - throw std::invalid_argument("BufferedRowConverter::get"); - } - if (numbits != 0) { - ret &= bitmask(numbits); - ret >>= bit_cursor; - } - return ret; - } - void set(unsigned long long val, int width, int numbits = 0) { - if (numbits != 0) { - val <<= bit_cursor; - val &= bitmask(numbits); - val |= (get(width) & ~bitmask(numbits)); - } - if (byte_cursor + abs(width) > buffer.size()) { - throw std::out_of_range("BufferedRowConverter::set"); - } - switch (width) { - case 1: - to_array(buffer, val, byte_cursor); - break; - case 2: - to_array(buffer, val, byte_cursor); - break; - case 4: - to_array(buffer, val, byte_cursor); - break; - case 8: - to_array(buffer, val, byte_cursor); - break; - case -1: - to_array(buffer, val, byte_cursor); - break; - case -2: - to_array(buffer, val, byte_cursor); - break; - case -4: - to_array(buffer, val, byte_cursor); - break; - case -8: - to_array(buffer, val, byte_cursor); - break; - default: - throw std::invalid_argument("BufferedRowConverter::set"); - } - } - void advance(int nbytes, int nbits = 0) { - if (nbits != 0) { - bit_cursor += nbits; - if (bit_cursor >= nbytes * 8) { - byte_cursor += nbytes; - bit_cursor -= nbytes * 8; - } - } else { - byte_cursor += nbytes; - bit_cursor = 0; - } - } - void carriage_return() { - byte_cursor = 0; - bit_cursor = 0; - buffer.assign(buffer.size(), 0); - } -}; - -#endif //GUARD_MANIFEST_H diff --git a/tools/csv2bin/Options.cpp b/tools/csv2bin/Options.cpp deleted file mode 100644 index 5ba123055e..0000000000 --- a/tools/csv2bin/Options.cpp +++ /dev/null @@ -1,273 +0,0 @@ -#include "Options.h" - -template -V& to_upper(V& vec) { - std::for_each(vec.begin(), vec.end(), [](typename V::value_type &x) { x = std::toupper(x); }); - return vec; -} - -void Options::short_usage(std::ostream &strm) { - strm << "Usage: csv2bin [OPTIONS] MODE CSV BIN MANIFEST" << std::endl; -} - -void Options::usage(std::ostream &strm) { - short_usage(strm); - strm << std::endl; - strm << R"( MODE Either "compile" or "disasm", controls)" << std::endl; - strm << " the mode of execution." << std::endl; - strm << " CSV Path to the CSV file. If compiling, the file" << std::endl; - strm << " must exist." << std::endl; - strm << " BIN Path to the compiled binary. If disassembling," << std::endl; - strm << " the file must exist." << std::endl; - strm << " MANIFEST Column specification for the CSV file. Not all" << std::endl; - strm << " columns need to be specified, but missing columns" << std::endl; - strm << " are presumed to be u32." << std::endl; - strm << " -i PATH Add header search paths. Can repeat as much as you want." << std::endl; - strm << " --include PATH Alias for -i" << std::endl; - strm << " --narc Output a NARC file" << std::endl; - strm << " --naix Output a NAIX file. Implies --narc" << std::endl; - strm << " --pad PADVAL Pads with PADVAL between rows (default: 0)" << std::endl; - strm << " -h Prints this message and exits" << std::endl; - strm << std::endl; - strm << "Note: Manifest file format declares each column on a separate line." << std::endl; - strm << "Columns must be declared in the same order as they would appear in" << std::endl; - strm << "both the CSV and the corresponding field in the compiled binary" << std::endl; - strm << "Each line shall have the following format:" << std::endl; - strm << std::endl; - strm << "column_name:width[.bits][:path/to/c/header.h[:const-prefix]]" << std::endl; - strm << std::endl; - strm << "column_name must match the CSV header line." << std::endl; - strm << "width must be a valid Nitro type name - either 'u' or 's'," << std::endl; - strm << " followed by either 8, 16, 32, or 64." << std::endl; - strm << " To designate a column as an index column, use 'skip'." << std::endl; - strm << " To designate a column as binary padding, use 'pad' followed by" << std::endl; - strm << " the number of padding bytes." << std::endl; - strm << " To designate a bitfield to a typed or padded field, add a period (.) followed" << std::endl; - strm << " by the number of bits. For example, 'can_cycle:u8.1'" << std::endl; - strm << "the optional header file path should be relative to" << std::endl; - strm << " the directory of execution, or you can pass a different" << std::endl; - strm << R"( root using "-i".)" << std::endl; - strm << " This spec supports the special header type 'bool', which maps" << std::endl; - strm << " the strings 'true' and 'false'." << std::endl; - strm << "the optional const-prefix restricts the header parsing to only those" << std::endl; - strm << " constants beginning with the specified prefix. Default behavior is to" << std::endl; - strm << " consider all constants defined in that header." << std::endl; - strm << std::endl; - strm << "Note 2: In disasm mode, the manifest must define all columns." << std::endl; - strm << "In compile mode, missing columns will be assumed to be u32." << std::endl; - strm << "The corresponding field will be inserted after the previous named" << std::endl; - strm << "column from the CSV." << std::endl; -} - -Options::Options(int argc, char **argv) { - std::vector argvec(argv + 1, argv + argc); - include_paths.insert(include_paths.begin(), "."); - for (auto iarg = argvec.begin(); iarg != argvec.end(); iarg ++) { - if (*iarg == "-i" || *iarg == "--include") { - iarg++; - if (iarg == argvec.end()) { - throw argument_error("missing argument to " + iarg[-1]); - } - assert(fs::exists(*iarg)); - include_paths.emplace_back(*iarg); - } else if (*iarg == "-h" || *iarg == "--help") { - usage(); - std::exit(EXIT_SUCCESS); - } else if (*iarg == "--narc") { - narc_mode = true; - } else if (*iarg == "--naix") { - naix_mode = true; - narc_mode = true; - } else if (*iarg == "--pad") { - iarg++; - padval = std::stoi(*iarg, 0, 0); - } else if ((*iarg)[0] == '-') { - throw argument_error("caught invalid option flag: " + *iarg); - } else { - posargs.emplace_back(*iarg); - } - } - if (posargs.size() < 4) { - throw argument_error("missing required argument(s)"); - } - assert(posargs.size() >= 4); - if (posargs[0] == "compile") { - execMode = EXEC_CSV2BIN; - } else if (posargs[0] == "disasm") { - if (naix_mode) { - std::cerr << "csv2bin warning: --naix with disasm mode is equivalent to --narc" << std::endl; - } - execMode = EXEC_BIN2CSV; - } else { - throw argument_error(R"(first positional argument must be either "compile" or "disasm", not )" + posargs[0]); - } - switch (execMode) { - case EXEC_CSV2BIN: - binfile.out = new std::ofstream(posargs[2], std::ios::binary); - break; - case EXEC_BIN2CSV: - binfile.in = new std::ifstream(posargs[2], narc_mode ? std::ios::binary : std::ios::binary | std::ios::ate); - break; - default: - assert(0); - } - manifest.read(posargs[3], include_paths); - if (execMode == EXEC_CSV2BIN) { - csvFile.FromFile(posargs[1]); - } -} - -Options::~Options() { - switch (execMode) { - case EXEC_CSV2BIN: - delete binfile.out; - break; - case EXEC_BIN2CSV: - delete binfile.in; - break; - default: - assert(0); - } -} - -int Options::main() { - switch (execMode) { - case EXEC_CSV2BIN: - return main_compile(); - case EXEC_BIN2CSV: - return main_disasm(); - default: - return 1; - } -} - -int Options::main_compile() { - for (auto name_i = csvFile.GetColnames().cbegin(); name_i != csvFile.GetColnames().cend(); name_i ++) { - if (!manifest[*name_i].is_init()) { - manifest[*name_i] = ColumnSpec(sizeof(unsigned int)); - if (name_i == csvFile.GetColnames().cbegin()) { - manifest.colnames.insert(manifest.colnames.cbegin(), *name_i); - } else { - auto dest_i = std::find(manifest.colnames.cbegin(), manifest.colnames.cend(), name_i[-1]); - manifest.colnames.insert(dest_i + 1, *name_i); - } - } - } - - if (narc_mode) { - fs::path naixfilename; - fs::path narcfilename; - std::ofstream naixfile; - std::string guard, guard_sub; - if (naix_mode) { - static const char do_not_modify[] = "/*\n" - " * THIS FILE WAS AUTOMATICALLY\n" - " * GENERATED BY tools/csv2bin\n" - " * DO NOT MODIFY!!!\n" - " */\n"; - narcfilename = posargs[2]; - naixfilename = narcfilename.replace_extension("naix"); - naixfile.open(naixfilename, std::ios::out); - naixfile << do_not_modify << std::endl; - guard_sub = narcfilename.replace_extension().filename().string(); - guard = "NARC_" + guard_sub + "_NAIX_"; - to_upper(guard); - naixfile << "#ifndef " << guard << std::endl; - naixfile << "#define " << guard << std::endl; - naixfile << std::endl; - naixfile << "enum {" << std::endl; - } - unsigned gmif_size = manifest.size() * csvFile.nrow() + 8; - unsigned btaf_size = 8 * csvFile.nrow() + 12; - unsigned btnf_size = 16; - auto *narc_header = new unsigned char[16]; - auto *gmif = new unsigned char[8]; - auto *btnf = new unsigned char[btnf_size]; - auto *btaf = new unsigned char[btaf_size]; - memcpy(narc_header, "NARC\xFE\xFF\x00\x01\x00\x00\x00\x00\x10\x00\x03\x00", 16); - to_array(narc_header, gmif_size + btaf_size + btnf_size + 16, 8); - memcpy(btaf, "BTAF", 4); - to_array(btaf, btaf_size, 4); - to_array(btaf, csvFile.nrow(), 8); - for (int i = 0; i < csvFile.nrow(); i++) { - to_array(btaf, i * manifest.size(), 12 + 8 * i); - to_array(btaf, i * manifest.size() + manifest.size(0), 16 + 8 * i); - if (naix_mode) { - char num_buf[10] = {0}; - sprintf(num_buf, "%04d", i); - naixfile << " NARC_" << guard_sub << "_" << num_buf << "_bin = " << i << "," << std::endl; - } - } - memcpy(btnf, "BTNF\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x01\x00", 16); - memcpy(gmif, "GMIF", 4); - to_array(gmif, gmif_size, 4); - binfile.out->write((const char *)narc_header, 16); - binfile.out->write((const char *)btaf, btaf_size); - binfile.out->write((const char *)btnf, btnf_size); - binfile.out->write((const char *)gmif, 8); - delete[] gmif; - delete[] btnf; - delete[] btaf; - delete[] narc_header; - if (naix_mode) { - naixfile << "};" << std::endl; - naixfile << std::endl; - naixfile << "#endif // " << guard << std::endl; - naixfile.close(); - } - } - - BufferedRowConverter converter(manifest, csvFile, padval); - for (const auto &row : csvFile) { - *binfile.out << converter; - } - return 0; -} - -int Options::main_disasm() { - std::vector colnames; - // pad cols are skipped from the csv but not the binary - for (auto name_i = manifest.colnames.begin(); name_i != manifest.colnames.end(); name_i ++) { - if (!manifest[*name_i].is_padding()) { - colnames.emplace_back(*name_i); - } - } - size_t binfsize; - size_t manifest_size = manifest.size(); - if (narc_mode) { - static unsigned char narc_header[16]; - static unsigned char btaf_header[12]; - binfile.in->read((char *)narc_header, 16); - assert(memcmp(narc_header, "NARC\xfe\xff\x00\x01", 8) == 0); - binfile.in->read((char *)btaf_header, 12); - assert(memcmp(btaf_header, "BTAF", 4) == 0); - size_t fatb_size = from_array(btaf_header, 4) - 12; - size_t nrow = from_array(btaf_header, 8); - binfile.in->seekg(fatb_size, std::ios::cur); - static unsigned char btnf_header[8]; - binfile.in->read((char *)btnf_header, 8); - assert(memcmp(btnf_header, "BTNF", 4) == 0); - size_t fntb_size = from_array(btnf_header, 4) - 8; - binfile.in->seekg(fntb_size, std::ios::cur); - static unsigned char gmif_header[8]; - binfile.in->read((char *)gmif_header, 8); - assert(memcmp(gmif_header, "GMIF", 4) == 0); - binfsize = from_array(gmif_header, 4) - 8; - assert(binfsize / manifest_size == nrow); - } else { - // binfile was opened at end (ios::ate) - binfsize = binfile.in->tellg(); - // rewind - binfile.in->seekg(0); - } - assert(binfsize % manifest_size == 0); - csvFile.resize(binfsize / manifest_size, colnames.size()); - csvFile.SetColnames(colnames.cbegin(), colnames.cend()); - - BufferedRowConverter converter(manifest, csvFile, padval); - for (int i = 0; i < csvFile.nrow(); i++) { - *binfile.in >> converter; - } - csvFile.ToFile(posargs[1]); - return 0; -} diff --git a/tools/csv2bin/Options.h b/tools/csv2bin/Options.h deleted file mode 100644 index 2ecfa3a681..0000000000 --- a/tools/csv2bin/Options.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef GUARD_OPTIONS_H -#define GUARD_OPTIONS_H - -#include "global.h" -#include "CsvFile.h" -#include "Manifest.h" - -enum ExecMode { - EXEC_UNSET = 0, - EXEC_CSV2BIN = 1, - EXEC_BIN2CSV = 2, -}; - -class argument_error : public std::runtime_error { -public: - explicit argument_error(const char *s) : std::runtime_error(s) {} - explicit argument_error(const std::string &s) : std::runtime_error(s) {} -}; - -class Options { - ExecMode execMode = EXEC_UNSET; - union { - std::ifstream *in; - std::ofstream *out; - } binfile; - CsvFile csvFile; - Manifest manifest; - std::vector include_paths; - std::vector posargs; - bool narc_mode = false; - bool naix_mode = false; - unsigned char padval = 0; - - int main_compile(); - int main_disasm(); -public: - static void short_usage(std::ostream &strm = std::cout); - static void usage(std::ostream &strm = std::cout); - Options(int argc, char ** argv); - ~Options(); - int main(); -}; - - -#endif //GUARD_OPTIONS_H diff --git a/tools/csv2bin/global.h b/tools/csv2bin/global.h deleted file mode 100644 index 3ab4779448..0000000000 --- a/tools/csv2bin/global.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef GUARD_GLOBAL_H -#define GUARD_GLOBAL_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if (__cplusplus < 201703L) -#include -namespace fs = std::experimental::filesystem; -#else -#include -namespace fs = std::filesystem; -#endif - -#endif //GUARD_GLOBAL_H diff --git a/tools/csv2bin/main.cpp b/tools/csv2bin/main.cpp deleted file mode 100644 index 5525d07bc4..0000000000 --- a/tools/csv2bin/main.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include "Options.h" - -int main(int argc, char ** argv) { -#ifdef NDEBUG - try { -#endif - return Options(argc, argv).main(); -#ifdef NDEBUG - } catch (std::bad_alloc const &e ) { - std::cerr << "csv2bin fatal error: bad alloc: " << e.what() << std::endl; - return 1; - } catch (argument_error const &e ) { - Options::short_usage(std::cerr); - std::cerr << "csv2bin argument error: " << e.what() << std::endl; - return 1; - } catch (std::exception const &e) { - std::cerr << "csv2bin error: " << e.what() << std::endl; - return 1; - } -#endif -} diff --git a/tools/csv2bin/meson.build b/tools/csv2bin/meson.build deleted file mode 100644 index 3b23fd22df..0000000000 --- a/tools/csv2bin/meson.build +++ /dev/null @@ -1,13 +0,0 @@ -csv2bin_exe = executable('csv2bin', - sources: [ - 'main.cpp', - 'CsvFile.cpp', - 'Manifest.cpp', - 'Options.cpp' - ], - cpp_args: [ - '-DNDEBUG', - '-std=c++17' - ], - native: true -) diff --git a/tools/meson.build b/tools/meson.build index d006e7d5fd..28c159644b 100644 --- a/tools/meson.build +++ b/tools/meson.build @@ -7,7 +7,6 @@ subdir('dataproc') # Requires libnitroarc and libenum subdir('msgenc') # Requires libdataproc # Other native tools -subdir('csv2bin') subdir('datagen') subdir('fixrom') subdir('jsoncnv')