Leahnaya/bemaniutils
1
0
Fork 0
mirror of https://github.com/DragonMinded/bemaniutils.git synced 2026-04-22 00:37:15 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

iidx: Support cannonballers and rootage.

Browse Source 
Add stubs for heroic verse and bistrover
This commit is contained in:
seth 2021-05-17 21:51:26 +00:00 committed by Jennifer Taylor
parent 4584cb3f45
commit c77d834091
8 changed files with 4757 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bemani/backend/iidx/bistrover.py Normal file
View File

@ -0,0 +1,15 @@
# vim: set fileencoding=utf-8
from typing import Optional
from bemani.backend.iidx.base import IIDXBase
from bemani.backend.iidx.heroicverse import IIDXHeroicVerse
from bemani.common import VersionConstants
class IIDXBistrover(IIDXBase):
name = 'Beatmania IIDX BISTROVER'
version = VersionConstants.IIDX_BISTROVER
def previous_version(self) -> Optional[IIDXBase]:
return IIDXHeroicVerse(self.data, self.config, self.model)

2226
bemani/backend/iidx/cannonballers.py
View File

File diff suppressed because it is too large Load Diff

28
bemani/backend/iidx/factory.py
View File

@ -28,7 +28,10 @@ from bemani.backend.iidx.pendual import IIDXPendual
from bemani.backend.iidx.copula import IIDXCopula
from bemani.backend.iidx.sinobuz import IIDXSinobuz
from bemani.backend.iidx.cannonballers import IIDXCannonBallers
from bemani.common import Model, VersionConstants
from bemani.backend.iidx.rootage import IIDXRootage
from bemani.backend.iidx.heroicverse import IIDXHeroicVerse
from bemani.backend.iidx.bistrover import IIDXBistrover
from bemani.common import Model, VersionConstants, GameConstants
from bemani.data import Data
@ -60,6 +63,9 @@ class IIDXFactory(Factory):
IIDXCopula,
IIDXSinobuz,
IIDXCannonBallers,
IIDXRootage,
IIDXHeroicVerse,
IIDXBistrover,
]
@classmethod
@ -81,8 +87,14 @@ class IIDXFactory(Factory):
return VersionConstants.IIDX_COPULA
if date >= 2016102600 and date < 2017122100:
return VersionConstants.IIDX_SINOBUZ
if date >= 2017122100:
if date >= 2017122100 and date < 2018110700:
return VersionConstants.IIDX_CANNON_BALLERS
if date >= 2018110700 and date < 2019101600:
return VersionConstants.IIDX_ROOTAGE
if date >= 2019101600 and date < 2020102800:
return VersionConstants.IIDX_HEROIC_VERSE
if date >= 2020102800:
return VersionConstants.IIDX_BISTROVER
return None
if model.game == 'JDJ':
@ -111,6 +123,12 @@ class IIDXFactory(Factory):
return IIDXCopula(data, config, model)
if parentversion == VersionConstants.IIDX_CANNON_BALLERS:
return IIDXSinobuz(data, config, model)
if parentversion == VersionConstants.IIDX_ROOTAGE:
return IIDXCannonBallers(data, config, model)
if parentversion == VersionConstants.IIDX_HEROIC_VERSE:
return IIDXRootage(data, config, model)
if parentversion == VersionConstants.IIDX_BISTROVER:
return IIDXHeroicVerse(data, config, model)
# Unknown older version
return None
@ -128,6 +146,12 @@ class IIDXFactory(Factory):
return IIDXSinobuz(data, config, model)
if version == VersionConstants.IIDX_CANNON_BALLERS:
return IIDXCannonBallers(data, config, model)
if version == VersionConstants.IIDX_ROOTAGE:
return IIDXRootage(data, config, model)
if version == VersionConstants.IIDX_HEROIC_VERSE:
return IIDXHeroicVerse(data, config, model)
if version == VersionConstants.IIDX_BISTROVER:
return IIDXBistrover(data, config, model)
# Unknown game version
return None

15
bemani/backend/iidx/heroicverse.py Normal file
View File

@ -0,0 +1,15 @@
# vim: set fileencoding=utf-8
from typing import Optional
from bemani.backend.iidx.base import IIDXBase
from bemani.backend.iidx.rootage import IIDXRootage
from bemani.common import VersionConstants
class IIDXHeroicVerse(IIDXBase):
name = 'Beatmania IIDX HEROIC VERSE'
version = VersionConstants.IIDX_HEROIC_VERSE
def previous_version(self) -> Optional[IIDXBase]:
return IIDXRootage(self.data, self.config, self.model)

2164
bemani/backend/iidx/rootage.py Normal file
View File

File diff suppressed because it is too large Load Diff

3
bemani/common/constants.py
View File

@ -59,6 +59,9 @@ class VersionConstants:
IIDX_COPULA: Final[int] = 23
IIDX_SINOBUZ: Final[int] = 24
IIDX_CANNON_BALLERS: Final[int] = 25
IIDX_ROOTAGE: Final[int] = 26
IIDX_HEROIC_VERSE: Final[int] = 27
IIDX_BISTROVER: Final[int] = 28
JUBEAT: Final[int] = 1
JUBEAT_RIPPLES: Final[int] = 2

6
bemani/format/iidxmusicdb.py
View File

@ -80,6 +80,12 @@ class IIDXMusicDB:
elif data[4] == 0x18:
offset = 0xc360
leap = 0x340
elif data[4] == 0x19:
offset = 0xCB30
leap = 0x340
elif data[4] == 0x1A:
offset = 0xD300
leap = 0x344
if sig[0] != b'IIDX':
raise Exception(f'Invalid signature \'{sig[0]}\' found!')

370
bemani/utils/read.py
View File

@ -1470,29 +1470,33 @@ class ImportIIDX(ImportBase):
no_combine: bool,
update: bool,
) -> None:
if version in ['20', '21', '22', '23', '24']:
if version in ['20', '21', '22', '23', '24', '25', '26']:
actual_version = {
'20': VersionConstants.IIDX_TRICORO,
'21': VersionConstants.IIDX_SPADA,
'22': VersionConstants.IIDX_PENDUAL,
'23': VersionConstants.IIDX_COPULA,
'24': VersionConstants.IIDX_SINOBUZ,
'25': VersionConstants.IIDX_CANNON_BALLERS,
'26': VersionConstants.IIDX_ROOTAGE,
}[version]
self.charts = [0, 1, 2, 3, 4, 5, 6]
elif version in ['omni-20', 'omni-21', 'omni-22', 'omni-23', 'omni-24']:
elif version in ['omni-20', 'omni-21', 'omni-22', 'omni-23', 'omni-24', 'omni-25', 'omni-26']:
actual_version = {
'omni-20': VersionConstants.IIDX_TRICORO,
'omni-21': VersionConstants.IIDX_SPADA,
'omni-22': VersionConstants.IIDX_PENDUAL,
'omni-23': VersionConstants.IIDX_COPULA,
'omni-24': VersionConstants.IIDX_SINOBUZ,
'omni-25': VersionConstants.IIDX_CANNON_BALLERS,
'omni-26': VersionConstants.IIDX_ROOTAGE,
}[version] + DBConstants.OMNIMIX_VERSION_BUMP
self.charts = [0, 1, 2, 3, 4, 5, 6]
elif version == 'all':
actual_version = None
self.charts = [0, 1, 2, 3, 4, 5, 6]
else:
raise Exception("Unsupported IIDX version, expected one of the following: 20, 21, 22, 23, 24, omni-20, omni-21, omni-22, omni-23, omni-24!")
raise Exception("Unsupported IIDX version, expected one of the following: 20, 21, 22, 23, 24, 25, 26, omni-20, omni-21, omni-22, omni-23, omni-24, omni-25, omni-26!")
super().__init__(config, GameConstants.IIDX, actual_version, no_combine, update)
@ -1545,6 +1549,18 @@ class ImportIIDX(ImportBase):
16212: 21066,
22096: 23030,
22097: 23051,
21214: 11101,
21221: 14101,
21225: 15104,
21226: 15102,
21231: 15101,
21237: 15103,
21240: 16105,
21242: 16104,
21253: 16103,
21258: 16102,
21262: 16101,
21220: 14100,
}
# Some charts were changed, and others kept the same on these
if chart in [0, 1, 2]:
@ -1581,6 +1597,18 @@ class ImportIIDX(ImportBase):
21066: 16212,
23030: 22096,
23051: 22097,
11101: 21214,
14101: 21221,
15104: 21225,
15102: 21226,
15101: 21231,
15103: 21237,
16105: 21240,
16104: 21242,
16103: 21253,
16102: 21258,
16101: 21262,
14100: 21220,
}
# Some charts were changed, and others kept the same on tehse
if chart in [0, 1, 2]:
@ -1605,7 +1633,7 @@ class ImportIIDX(ImportBase):
return 1
return chart
def scrape(self, binfile: str, assets_dir: Optional[str]) -> List[Dict[str, Any]]:
def scrape(self, binfile: str, assets_dir: Optional[str]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
if self.version is None:
raise Exception('Can\'t import IIDX database for \'all\' version!')
@ -1620,73 +1648,250 @@ class ImportIIDX(ImportBase):
finally:
bh.close()
musicdb = IIDXMusicDB(binarydata)
import_qpros = True # by default, try to import qpros
try:
pe = pefile.PE(data=binarydata, fast_load=True)
except BaseException:
import_qpros = False # if it failed then we're reading a music db file, not the executable
def virtual_to_physical(offset: int) -> int:
for section in pe.sections:
start = section.VirtualAddress + pe.OPTIONAL_HEADER.ImageBase
end = start + section.SizeOfRawData
if offset >= start and offset < end:
return (offset - start) + section.PointerToRawData
raise Exception(f'Couldn\'t find raw offset for virtual offset 0x{offset:08x}')
songs: List[Dict[str, Any]] = []
for song in musicdb.songs:
bpm = (0, 0)
notecounts = [0, 0, 0, 0, 0, 0]
if not import_qpros:
musicdb = IIDXMusicDB(binarydata)
for song in musicdb.songs:
bpm = (0, 0)
notecounts = [0, 0, 0, 0, 0, 0]
if song.id in self.BANNED_CHARTS:
continue
if song.id in self.BANNED_CHARTS:
continue
if sound_files is not None:
if song.id in sound_files:
# Look up chart info!
filename = sound_files[song.id]
_, extension = os.path.splitext(filename)
data = None
if sound_files is not None:
if song.id in sound_files:
# Look up chart info!
filename = sound_files[song.id]
_, extension = os.path.splitext(filename)
data = None
if extension == '.1':
fp = open(filename, 'rb')
data = fp.read()
fp.close()
if extension == '.1':
fp = open(filename, 'rb')
data = fp.read()
fp.close()
else:
fp = open(filename, 'rb')
ifsdata = fp.read()
fp.close()
ifs = IFS(ifsdata)
for fn in ifs.filenames:
_, extension = os.path.splitext(fn)
if extension == '.1':
data = ifs.read_file(fn)
if data is not None:
iidxchart = IIDXChart(data)
bpm_min, bpm_max = iidxchart.bpm
bpm = (bpm_min, bpm_max)
notecounts = iidxchart.notecounts
else:
print(f"Could not find chart information for song {song.id}!")
else:
fp = open(filename, 'rb')
ifsdata = fp.read()
fp.close()
ifs = IFS(ifsdata)
for fn in ifs.filenames:
_, extension = os.path.splitext(fn)
if extension == '.1':
data = ifs.read_file(fn)
print(f"No chart information because chart for song {song.id} is missing!")
songs.append({
'id': song.id,
'title': song.title,
'artist': song.artist,
'genre': song.genre,
'bpm_min': bpm[0],
'bpm_max': bpm[1],
'difficulty': {
'spn': song.difficulties[0],
'sph': song.difficulties[1],
'spa': song.difficulties[2],
'dpn': song.difficulties[3],
'dph': song.difficulties[4],
'dpa': song.difficulties[5],
},
'notecount': {
'spn': notecounts[0],
'sph': notecounts[1],
'spa': notecounts[2],
'dpn': notecounts[3],
'dph': notecounts[4],
'dpa': notecounts[5],
},
})
if data is not None:
iidxchart = IIDXChart(data)
bpm_min, bpm_max = iidxchart.bpm
bpm = (bpm_min, bpm_max)
notecounts = iidxchart.notecounts
else:
print(f"Could not find chart information for song {song.id}!")
qpros: List[Dict[str, Any]] = []
if self.version == VersionConstants.IIDX_TRICORO:
stride = 4
qp_head_offset = 0x1CCB18 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 79 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x1CCC58
qp_hair_length = 103
qp_face_offset = 0x1CCDF8
qp_face_length = 50
qp_hand_offset = 0x1CCEC0
qp_hand_length = 103
qp_body_offset = 0x1CD060
qp_body_length = 106
filename_offset = 0
packedfmt = (
'I' # filename
)
if self.version == VersionConstants.IIDX_SPADA:
stride = 4
qp_head_offset = 0x213B50 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 125 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x213D48
qp_hair_length = 126
qp_face_offset = 0x213F40
qp_face_length = 72
qp_hand_offset = 0x214060
qp_hand_length = 135
qp_body_offset = 0x214280
qp_body_length = 135
filename_offset = 0
packedfmt = (
'I' # filename
)
if self.version == VersionConstants.IIDX_PENDUAL:
stride = 4
qp_head_offset = 0x1D5228 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 163 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x1D54B8
qp_hair_length = 182
qp_face_offset = 0x1D5790
qp_face_length = 106
qp_hand_offset = 0x1D5938
qp_hand_length = 184
qp_body_offset = 0x1D5C18
qp_body_length = 191
filename_offset = 0
packedfmt = (
'I' # filename
)
if self.version == VersionConstants.IIDX_COPULA:
stride = 8
qp_head_offset = 0x12F9D8 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 186 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x12FFA8
qp_hair_length = 202
qp_face_offset = 0x1305F8
qp_face_length = 126
qp_hand_offset = 0x1309E8
qp_hand_length = 206
qp_body_offset = 0x131058
qp_body_length = 211
filename_offset = 0
qpro_id_offset = 1
packedfmt = (
'I' # filename
'I' # string containing id and name of the part
)
if self.version == VersionConstants.IIDX_SINOBUZ:
stride = 8
qp_head_offset = 0x149F88 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 211 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x14A620
qp_hair_length = 245
qp_face_offset = 0x14ADC8
qp_face_length = 152
qp_hand_offset = 0x14B288
qp_hand_length = 236
qp_body_offset = 0x14B9E8
qp_body_length = 256
filename_offset = 0
qpro_id_offset = 1
packedfmt = (
'I' # filename
'I' # string containing id and name of the part
)
if self.version == VersionConstants.IIDX_CANNON_BALLERS:
stride = 16
qp_head_offset = 0x2339E0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 231 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x234850
qp_hair_length = 267
qp_face_offset = 0x235900
qp_face_length = 173
qp_hand_offset = 0x2363D0
qp_hand_length = 261
qp_body_offset = 0x237420
qp_body_length = 282
filename_offset = 0
qpro_id_offset = 1
packedfmt = (
'L' # filename
'L' # string containing id and name of the part
)
if self.version == VersionConstants.IIDX_ROOTAGE:
stride = 16
qp_head_offset = 0x5065F0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 259 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x507620
qp_hair_length = 288
qp_face_offset = 0x508820
qp_face_length = 193
qp_hand_offset = 0x509430
qp_hand_length = 287
qp_body_offset = 0x50A620
qp_body_length = 304
filename_offset = 0
qpro_id_offset = 1
packedfmt = (
'L' # filename
'L' # string containing id and name of the part
)
def read_string(offset: int) -> str:
# First, translate load offset in memory to disk offset
offset = virtual_to_physical(offset)
# Now, grab bytes until we're null-terminated
bytestring = []
while binarydata[offset] != 0:
bytestring.append(binarydata[offset])
offset = offset + 1
# Its shift-jis encoded, so decode it now
return bytes(bytestring).decode('shift_jisx0213')
def read_qpro_db(offset: int, length: int, qp_type: str) -> None:
for qpro_id in range(length):
chunkoffset = offset + (stride * qpro_id)
chunkdata = binarydata[chunkoffset:(chunkoffset + stride)]
unpacked = struct.unpack(packedfmt, chunkdata)
filename = read_string(unpacked[filename_offset]).replace('qp_', '')
remove = f'_{qp_type}.ifs'
filename = filename.replace(remove, '').replace('_head1.ifs', '').replace('_head2.ifs', '')
if self.version in [VersionConstants.IIDX_TRICORO, VersionConstants.IIDX_SPADA, VersionConstants.IIDX_PENDUAL]:
name = filename # qpro names are not stored in these 3 games so use the identifier instead
else:
print(f"No chart information because chart for song {song.id} is missing!")
name = read_string(unpacked[qpro_id_offset])[4:] # qpro name is stored in second string of form "000:name"
qproinfo = {
'identifier': filename,
'id': qpro_id,
'name': name,
'type': qp_type,
}
qpros.append(qproinfo)
if import_qpros:
read_qpro_db(qp_head_offset, qp_head_length, 'head')
read_qpro_db(qp_hair_offset, qp_hair_length, 'hair')
read_qpro_db(qp_face_offset, qp_face_length, 'face')
read_qpro_db(qp_hand_offset, qp_hand_length, 'hand')
read_qpro_db(qp_body_offset, qp_body_length, 'body')
songs.append({
'id': song.id,
'title': song.title,
'artist': song.artist,
'genre': song.genre,
'bpm_min': bpm[0],
'bpm_max': bpm[1],
'difficulty': {
'spn': song.difficulties[0],
'sph': song.difficulties[1],
'spa': song.difficulties[2],
'dpn': song.difficulties[3],
'dph': song.difficulties[4],
'dpa': song.difficulties[5],
},
'notecount': {
'spn': notecounts[0],
'sph': notecounts[1],
'spa': notecounts[2],
'dpn': notecounts[3],
'dph': notecounts[4],
'dpa': notecounts[5],
},
})
return songs
return songs, qpros
def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
def lookup(self, server: str, token: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
if self.version is None:
raise Exception('Can\'t look up IIDX database for \'all\' version!')
@ -1741,7 +1946,18 @@ class ImportIIDX(ImportBase):
lut[song.id]['notecount'][chart_map[song.chart]] = song.data.get_int('notecount')
# Return the reassembled data
return [val for _, val in lut.items()]
qpros: List[Dict[str, Any]] = []
game = self.remote_game(server, token)
for item in game.get_items(self.game, self.version):
if 'qp_' in item.type:
qpros.append({
'identifier': item.data.get_str('identifier'),
'id': item.id,
'name': item.data.get_str('name'),
'type': item.data.get_str('type'),
})
return [val for _, val in lut.items()], qpros
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
if self.version is None:
@ -1782,6 +1998,29 @@ class ImportIIDX(ImportBase):
self.insert_music_id_for_song(next_id, song['id'], chart, song['title'], song['artist'], song['genre'], songdata)
self.finish_batch()
def import_qpros(self, qpros: List[Dict[str, Any]]) -> None:
if self.version is None:
raise Exception('Can\'t import IIDX database for \'all\' version!')
self.start_batch()
for i, qpro in enumerate(qpros):
# Make importing faster but still do it in chunks
if (i % 16) == 15:
self.finish_batch()
self.start_batch()
print(f"New catalog entry for {qpro['id']}")
self.insert_catalog_entry(
f"qp_{qpro['type']}",
qpro['id'],
{
'name': qpro['name'],
'identifier': qpro['identifier'],
},
)
self.finish_batch()
def import_metadata(self, tsvfile: str) -> None:
if self.version is not None:
raise Exception("Unsupported IIDX version, expected one of the following: all")
@ -3482,15 +3721,16 @@ if __name__ == "__main__":
else:
# Normal case, doing a music DB import.
if args.bin is not None:
songs = iidx.scrape(args.bin, args.assets)
songs, qpros = iidx.scrape(args.bin, args.assets)
elif args.server and args.token:
songs = iidx.lookup(args.server, args.token)
songs, qpros = iidx.lookup(args.server, args.token)
else:
raise Exception(
'No music_data.bin or TSV provided and no remote server specified! Please ' +
'provide either a --bin, --tsv or a --server and --token option!'
)
iidx.import_music_db(songs)
iidx.import_qpros(qpros)
iidx.close()
elif args.series == GameConstants.DDR: