From 9f4dfe468230949847d09882b7fbdc46f9a8a7f6 Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Fri, 24 Apr 2020 19:20:27 +0000 Subject: [PATCH] Teach network how to import/export Jubeat emblem catalog. --- README.md | 8 +++ bemani/api/objects/catalog.py | 29 ++++++++++ bemani/data/api/game.py | 17 ++++++ bemani/utils/read.py | 101 ++++++++++++++++++++++++++++++---- 4 files changed, 144 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b090dab..0f78e96 100644 --- a/README.md +++ b/README.md @@ -402,6 +402,14 @@ after importing all mixes: ./read --config config/server.yaml --series jubeat --version all --tsv data/jubeat.tsv ``` +For Jubeat Prop and later versions, you will also need to import the emblem DB, or emblems +will not work properly. An example is as follows: + +``` +./read --config config/server.yaml --series jubeat --version prop \ + --xml data/emblem-info/emblem-info.xml +``` + ### IIDX For IIDX, you will need the data directory of the mix you wish to support. The import diff --git a/bemani/api/objects/catalog.py b/bemani/api/objects/catalog.py index 3f1dbef..eefe52e 100644 --- a/bemani/api/objects/catalog.py +++ b/bemani/api/objects/catalog.py @@ -128,9 +128,38 @@ class CatalogObject(BaseObject): ], } + def __format_jubeat_extras(self) -> Dict[str, List[Dict[str, Any]]]: + # Gotta look up the unlock catalog + items = self.data.local.game.get_items(self.game, self.version) + + # Format it depending on the version + if self.version in { + VersionConstants.JUBEAT_PROP, + VersionConstants.JUBEAT_QUBELL, + VersionConstants.JUBEAT_CLAN, + }: + return { + "emblems": [ + { + "index": str(item.id), + "song": item.data.get_int("music_id"), + "layer": item.data.get_int("layer"), + "evolved": item.data.get_int("evolved"), + "rarity": item.data.get_int("rarity"), + "name": item.data.get_str("name"), + } + for item in items + if item.type == "emblem" + ], + } + else: + return {"emblems": []} + def __format_extras(self) -> Dict[str, List[Dict[str, Any]]]: if self.game == GameConstants.SDVX: return self.__format_sdvx_extras() + elif self.game == GameConstants.JUBEAT: + return self.__format_jubeat_extras() else: return {} diff --git a/bemani/data/api/game.py b/bemani/data/api/game.py index 6d7875a..bb4707c 100644 --- a/bemani/data/api/game.py +++ b/bemani/data/api/game.py @@ -25,6 +25,19 @@ class GlobalGameData(BaseGlobalData): {}, ) + def __translate_jubeat_emblems(self, entry: Dict[str, Any]) -> Item: + return Item( + "emblem", + int(entry["index"]), + { + "music_id": int(entry["song"]), + "layer": int(entry["layer"]), + "evolved": int(entry["evolved"]), + "rarity": int(entry["rarity"]), + "name": entry["name"], + }, + ) + def get_items(self, game: str, version: int) -> List[Item]: """ Given a game/userid, find all items in the catalog. @@ -51,6 +64,10 @@ class GlobalGameData(BaseGlobalData): "purchases": self.__translate_sdvx_song_unlock, "appealcards": self.__translate_sdvx_appealcard, }.get(catalogtype, None) + elif game == GameConstants.JUBEAT: + translation = { + "emblems": self.__translate_jubeat_emblems, + }.get(catalogtype, None) else: translation = None diff --git a/bemani/utils/read.py b/bemani/utils/read.py index 1161935..6b0a4b1 100644 --- a/bemani/utils/read.py +++ b/bemani/utils/read.py @@ -1239,15 +1239,22 @@ class ImportJubeat(ImportBase): super().__init__(config, GameConstants.JUBEAT, actual_version, no_combine, update) - def scrape(self, xmlfile: str) -> List[Dict[str, Any]]: + def scrape(self, xmlfile: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: if self.version is None: raise Exception('Can\'t scrape Jubeat database for \'all\' version!') - tree = ET.parse(xmlfile) - root = tree.getroot() + try: + # Probably UTF-8 music DB + tree = ET.parse(xmlfile) + root = tree.getroot() + except ValueError: + # Probably shift-jis emblems + with open(xmlfile, 'rb') as xmlhandle: + xmldata = xmlhandle.read().decode('shift_jisx0213') + root = ET.fromstring(xmldata) songs: List[Dict[str, Any]] = [] - for music_entry in root.find('body'): + for music_entry in root.find('body') or []: songid = int(music_entry.find('music_id').text) bpm_min = float(music_entry.find('bpm_min').text) bpm_max = float(music_entry.find('bpm_max').text) @@ -1278,9 +1285,34 @@ class ImportJubeat(ImportBase): 'extreme': difficulties[2], }, }) - return songs - def lookup(self, server: str, token: str) -> List[Dict[str, Any]]: + emblems: List[Dict[str, Any]] = [] + if self.version in { + VersionConstants.JUBEAT_PROP, + VersionConstants.JUBEAT_QUBELL, + VersionConstants.JUBEAT_CLAN, + }: + for emblem_entry in root.find('emblem_list') or []: + print(emblem_entry) + index = int(emblem_entry.find('index').text) + layer = int(emblem_entry.find('layer').text) + music_id = int(emblem_entry.find('music_id').text) + evolved = int(emblem_entry.find('evolved').text) + rarity = int(emblem_entry.find('rarity').text) + name = emblem_entry.find('name').text + + emblems.append({ + 'id': index, + 'layer': layer, + 'music_id': music_id, + 'evolved': evolved, + 'rarity': rarity, + 'name': name, + }) + + return songs, emblems + + def lookup(self, server: str, token: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: if self.version is None: raise Exception('Can\'t look up Jubeat database for \'all\' version!') @@ -1316,8 +1348,28 @@ class ImportJubeat(ImportBase): } lut[song.id]['difficulty'][chart_map[song.chart]] = song.data.get_int('difficulty') - # Return the reassembled data - return [val for _, val in lut.items()] + # Reassemble the data + reassembled_songs = [val for _, val in lut.items()] + + emblems: List[Dict[str, Any]] = [] + if self.version in { + VersionConstants.JUBEAT_PROP, + VersionConstants.JUBEAT_QUBELL, + VersionConstants.JUBEAT_CLAN, + }: + game = self.remote_game(server, token) + for item in game.get_items(self.game, self.version): + if item.type == "emblem": + emblems.append({ + 'id': item.id, + 'layer': item.data.get_int('layer'), + 'music_id': item.data.get_int('music_id'), + 'evolved': item.data.get_int('evolved'), + 'rarity': item.data.get_int('rarity'), + 'name': item.data.get_str('name'), + }) + + return reassembled_songs, emblems def import_music_db(self, songs: List[Dict[str, Any]]) -> None: if self.version is None: @@ -1356,6 +1408,32 @@ class ImportJubeat(ImportBase): self.insert_music_id_for_song(next_id, songid, chart, song['title'], song['artist'], song['genre'], data) self.finish_batch() + def import_emblems(self, emblems: List[Dict[str, Any]]) -> None: + if self.version is None: + raise Exception('Can\'t import Jubeat database for \'all\' version!') + + self.start_batch() + for i, emblem in enumerate(emblems): + # Make importing faster but still do it in chunks + if (i % 16) == 15: + self.finish_batch() + self.start_batch() + + print(f"New catalog entry for {emblem['music_id']}") + self.insert_catalog_entry( + 'emblem', + emblem['id'], + { + 'layer': emblem['layer'], + 'music_id': emblem['music_id'], + 'evolved': emblem['evolved'], + 'rarity': emblem['rarity'], + 'name': emblem['name'], + }, + ) + + self.finish_batch() + def import_metadata(self, tsvfile: str) -> None: if self.version is not None: raise Exception("Unsupported Jubeat version, expected one of the following: all") @@ -3381,17 +3459,18 @@ if __name__ == "__main__": # hand-populated since its not in the music DB. jubeat.import_metadata(args.tsv) else: - # Normal case, doing a music DB import. + # Normal case, doing a music DB or emblem import. if args.xml is not None: - songs = jubeat.scrape(args.xml) + songs, emblems = jubeat.scrape(args.xml) elif args.server and args.token: - songs = jubeat.lookup(args.server, args.token) + songs, emblems = jubeat.lookup(args.server, args.token) else: raise Exception( 'No music_info.xml or TSV provided and no remote server specified! Please ' + 'provide either a --xml, --tsv or a --server and --token option!' ) jubeat.import_music_db(songs) + jubeat.import_emblems(emblems) jubeat.close() elif args.series == GameConstants.IIDX: