From 1e84a50330e663fc6b9fe852a885f1f1c844950d Mon Sep 17 00:00:00 2001 From: Midorica Date: Mon, 6 Nov 2023 21:04:49 -0500 Subject: [PATCH 001/130] fixing again the render_POST for CXB --- titles/cxb/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/cxb/index.py b/titles/cxb/index.py index 0ef8667..0c38d55 100644 --- a/titles/cxb/index.py +++ b/titles/cxb/index.py @@ -103,7 +103,7 @@ class CxbServlet(resource.Resource): else: self.logger.info(f"Ready on port {self.game_cfg.server.port}") - def render_POST(self, request: Request, version: int, endpoint: str): + def render_POST(self, request: Request): version = 0 internal_ver = 0 func_to_find = "" From e88e1f82f88973d100cde4486b93a5013a5f32fb Mon Sep 17 00:00:00 2001 From: Midorica Date: Mon, 6 Nov 2023 21:55:05 -0500 Subject: [PATCH 002/130] fixing get_energy for CXB --- titles/cxb/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/cxb/base.py b/titles/cxb/base.py index 89e9cc3..4a92ca6 100644 --- a/titles/cxb/base.py +++ b/titles/cxb/base.py @@ -530,7 +530,6 @@ class CxbBase: profile = self.data.profile.get_profile_index(0, uid, self.version) data1 = profile["data"] p = self.data.item.get_energy(uid) - energy = p["energy"] if not p: self.data.item.put_energy(uid, 5) @@ -543,6 +542,7 @@ class CxbBase: } array = [] + energy = p["energy"] newenergy = int(energy) + 5 self.data.item.put_energy(uid, newenergy) From a497a9806dda5320949effc418bc5db86941e0d5 Mon Sep 17 00:00:00 2001 From: Midorica Date: Tue, 7 Nov 2023 22:40:15 -0500 Subject: [PATCH 003/130] fixing CXB render_POST --- titles/cxb/index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/titles/cxb/index.py b/titles/cxb/index.py index 0c38d55..4011537 100644 --- a/titles/cxb/index.py +++ b/titles/cxb/index.py @@ -111,7 +111,10 @@ class CxbServlet(resource.Resource): subcmd = "" req_url = request.uri.decode() url_split = req_url.split("/") - req_bytes = request.content.getvalue() + try: + req_bytes = request.content.getvalue() + except: + req_bytes = request.content.read().decode("utf-8") try: req_json: Dict = json.loads(req_bytes) From d400a0be4b0b0c604b00b6f8929b38f29a6914c7 Mon Sep 17 00:00:00 2001 From: Midorica Date: Fri, 16 Aug 2024 09:25:39 -0400 Subject: [PATCH 004/130] adding luminous to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 505f241..c792928 100644 --- a/readme.md +++ b/readme.md @@ -29,6 +29,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + NEW PLUS + SUN + SUN PLUS + + LUMINOUS + crossbeats REV. + Crossbeats REV. From eacd4a2f4388861bba8226dee113a2054340d8c7 Mon Sep 17 00:00:00 2001 From: daydensteve Date: Mon, 2 Sep 2024 20:00:59 -0400 Subject: [PATCH 005/130] Adding stock_tickets and stock_count chuni mods. Enables specified tickets to be auto-stocked on login --- docs/game_specific_info.md | 14 ++++++++------ example_config/chuni.yaml | 6 +++++- titles/chuni/base.py | 27 +++++++++++++++++++++------ titles/chuni/config.py | 12 ++++++++++++ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 33d5710..37561fc 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -80,12 +80,14 @@ The importer for Chunithm will import: Events, Music, Charge Items and Avatar Ac Config file is located in `config/chuni.yaml`. -| Option | Info | -|------------------|----------------------------------------------------------------------------------------------------------------| -| `news_msg` | If this is set, the news at the top of the main screen will be displayed (up to Chunithm Paradise Lost) | -| `name` | If this is set, all players that are not on a team will use this one by default. | -| `use_login_bonus`| This is used to enable the login bonuses | -| `crypto` | This option is used to enable the TLS Encryption | +| Option | Info | +|------------------|---------------------------------------------------------------------------------------------------------------------| +| `news_msg` | If this is set, the news at the top of the main screen will be displayed (up to Chunithm Paradise Lost) | +| `name` | If this is set, all players that are not on a team will use this one by default. | +| `use_login_bonus`| This is used to enable the login bonuses | +| `stock_tickets` | If this is set, specifies tickets to auto-stock at login. Format is a comma-delimited list of IDs. Defaults to None | +| `stock_count` | Ignored if stock_tickets is not specified. Number to stock of each ticket. Defaults to 99 | +| `crypto` | This option is used to enable the TLS Encryption | If you would like to use network encryption, add the keys to the `keys` section under `crypto`, where the key diff --git a/example_config/chuni.yaml b/example_config/chuni.yaml index 4855fa1..ca27fad 100644 --- a/example_config/chuni.yaml +++ b/example_config/chuni.yaml @@ -8,7 +8,11 @@ team: mods: use_login_bonus: True - + # stock_tickets allows specified ticket IDs to be auto-stocked at login. Format is a comma-delimited string of ticket IDs + # note: quanity is not refreshed on "continue" after set - only on subsequent login + stock_tickets: + stock_count: 99 + version: 11: rom: 2.00.00 diff --git a/titles/chuni/base.py b/titles/chuni/base.py index 9be4bf7..0e0adac 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -24,20 +24,35 @@ class ChuniBase: async def handle_game_login_api_request(self, data: Dict) -> Dict: """ - Handles the login bonus logic, required for the game because - getUserLoginBonus gets called after getUserItem and therefore the + Handles the login bonus and ticket stock logic, required for the game + because getUserLoginBonus gets called after getUserItem; therefore the items needs to be inserted in the database before they get requested. - Adds a bonusCount after a user logged in after 24 hours, makes sure - loginBonus 30 gets looped, only show the login banner every 24 hours, - adds the bonus to items (itemKind 6) + - Adds a stock for each specified ticket (itemKind 5) + - Adds a bonusCount after a user logged in after 24 hours, makes sure + loginBonus 30 gets looped, only show the login banner every 24 hours, + adds the bonus to items (itemKind 6) """ + user_id = data["userId"] + + # If we want to make certain tickets always available, stock them now + if self.game_cfg.mods.stock_tickets: + for ticket in self.game_cfg.mods.stock_tickets.split(","): + await self.data.item.put_item( + user_id, + { + "itemId": ticket.strip(), + "itemKind": 5, + "stock": self.game_cfg.mods.stock_count, + "isValid": True, + }, + ) + # ignore the login bonus if disabled in config if not self.game_cfg.mods.use_login_bonus: return {"returnCode": 1} - user_id = data["userId"] login_bonus_presets = await self.data.static.get_login_bonus_presets(self.version) for preset in login_bonus_presets: diff --git a/titles/chuni/config.py b/titles/chuni/config.py index 72329ec..dcdfce4 100644 --- a/titles/chuni/config.py +++ b/titles/chuni/config.py @@ -53,6 +53,18 @@ class ChuniModsConfig: self.__config, "chuni", "mods", "use_login_bonus", default=True ) + @property + def stock_tickets(self) -> str: + return CoreConfig.get_config_field( + self.__config, "chuni", "mods", "stock_tickets", default=None + ) + + @property + def stock_count(self) -> int: + return CoreConfig.get_config_field( + self.__config, "chuni", "mods", "stock_count", default=99 + ) + class ChuniVersionConfig: def __init__(self, parent_config: "ChuniConfig") -> None: From 73dda06413c009c6f661f21935aa30c3ad615945 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 5 Sep 2024 11:37:52 -0400 Subject: [PATCH 006/130] mai2: add warning about portrait uploading not being supported. #67 --- titles/mai2/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b3c6a1d..bbd074f 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -818,7 +818,8 @@ class Mai2Base: } async def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict: - self.logger.debug(data) + self.logger.warning("Portrait uploading not supported at this time.") + return {'returnCode': 0, 'apiName': 'UploadUserPortraitApi'} async def handle_upload_user_photo_api_request(self, data: Dict) -> Dict: if not self.game_config.uploads.photos or not self.game_config.uploads.photos_dir: From 944b80129b4f00f7fbf011ed5297027cba277bcc Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 5 Sep 2024 11:45:22 -0400 Subject: [PATCH 007/130] chuni: fix ultimate/worlds end chart reading, closes #63 --- titles/chuni/read.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/titles/chuni/read.py b/titles/chuni/read.py index db7435c..15557d4 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -35,11 +35,15 @@ class ChuniReader(BaseReader): if self.opt_dir is not None: data_dirs += self.get_data_directories(self.opt_dir) + + we_diff = "4" + if self.version >= ChuniConstants.VER_CHUNITHM_NEW: + we_diff = "5" for dir in data_dirs: self.logger.info(f"Read from {dir}") await self.read_events(f"{dir}/event") - await self.read_music(f"{dir}/music") + await self.read_music(f"{dir}/music", we_diff) await self.read_charges(f"{dir}/chargeItem") await self.read_avatar(f"{dir}/avatarAccessory") await self.read_login_bonus(f"{dir}/") @@ -138,7 +142,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert event {id}") - async def read_music(self, music_dir: str) -> None: + async def read_music(self, music_dir: str, we_diff: str = "4") -> None: for root, dirs, files in walk(music_dir): for dir in dirs: if path.exists(f"{root}/{dir}/Music.xml"): @@ -169,7 +173,7 @@ class ChuniReader(BaseReader): chart_type = MusicFumenData.find("type") chart_id = chart_type.find("id").text chart_diff = chart_type.find("str").text - if chart_diff == "WorldsEnd" and (chart_id == "4" or chart_id == "5"): # 4 in SDBT, 5 in SDHD + if chart_diff == "WorldsEnd" and chart_id == we_diff: # 4 in SDBT, 5 in SDHD level = float(xml_root.find("starDifType").text) we_chara = ( xml_root.find("worldsEndTagName") From d1048694d40b5838ba7143a826ce6b8f77b405f0 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Fri, 6 Sep 2024 10:36:57 -0400 Subject: [PATCH 008/130] Fix --config option not being respected, fixes #172 --- core/__init__.py | 2 -- core/app.py | 3 ++- dbutils.py | 5 +++-- index.py | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index f5e306e..7b4e5d8 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,7 +1,5 @@ from core.config import CoreConfig -from core.allnet import AllnetServlet, BillingServlet from core.aimedb import AimedbServlette from core.title import TitleServlet from core.utils import Utils from core.mucha import MuchaServlet -from core.frontend import FrontendServlet diff --git a/core/app.py b/core/app.py index e4d3330..fa1c8f2 100644 --- a/core/app.py +++ b/core/app.py @@ -9,7 +9,8 @@ from starlette.responses import PlainTextResponse from os import environ, path, mkdir, W_OK, access from typing import List -from core import CoreConfig, TitleServlet, MuchaServlet, AllnetServlet, BillingServlet, AimedbServlette +from core import CoreConfig, TitleServlet, MuchaServlet +from core.allnet import AllnetServlet, BillingServlet from core.frontend import FrontendServlet async def dummy_rt(request: Request): diff --git a/dbutils.py b/dbutils.py index 21b5c9d..9314f8e 100644 --- a/dbutils.py +++ b/dbutils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse import logging -from os import mkdir, path, access, W_OK +from os import mkdir, path, access, W_OK, environ import yaml import asyncio @@ -25,10 +25,11 @@ if __name__ == "__main__": parser.add_argument("action", type=str, help="create, upgrade, downgrade, create-owner, migrate, create-revision, create-autorevision") args = parser.parse_args() + environ["ARTEMIS_CFG_DIR"] = args.config + cfg = CoreConfig() if path.exists(f"{args.config}/core.yaml"): cfg_dict = yaml.safe_load(open(f"{args.config}/core.yaml")) - cfg_dict.get("database", {})["loglevel"] = "info" cfg.update(cfg_dict) if not path.exists(cfg.server.log_dir): diff --git a/index.py b/index.py index 40a1bbd..2b755b0 100644 --- a/index.py +++ b/index.py @@ -6,7 +6,8 @@ import uvicorn import logging import asyncio -from core import CoreConfig, AimedbServlette +from core.config import CoreConfig +from core.aimedb import AimedbServlette async def launch_main(cfg: CoreConfig, ssl: bool) -> None: if ssl: From 1db020b5fc754da7d829338c4dc7425357c04a3d Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 9 Sep 2024 16:53:14 +0000 Subject: [PATCH 009/130] fix generated keychip validation failures --- core/adb_handlers/base.py | 2 +- core/data/schema/arcade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/adb_handlers/base.py b/core/adb_handlers/base.py index 06b5267..b5520ce 100644 --- a/core/adb_handlers/base.py +++ b/core/adb_handlers/base.py @@ -120,7 +120,7 @@ class ADBHeader: if self.store_id == 0: raise ADBHeaderException(f"Store ID cannot be 0!") - if re.fullmatch(r"^A[0-9]{2}[E|X][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None: + if re.fullmatch(r"^A[0-9]{2}[A-Z][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None: raise ADBHeaderException(f"Keychip ID {self.keychip_id} is invalid!") return True diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index 3e83bc5..5b570a1 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -232,7 +232,7 @@ class ArcadeData(BaseData): return f"{platform_code}{'-' if dash else ''}{platform_rev:02d}{serial_letter}{serial_num:04d}{append:04d}" def validate_keychip_format(self, serial: str) -> bool: - # For the 2nd letter, E and X are the only "real" values that have been observed + # For the 2nd letter, E and X are the only "real" values that have been observed (A is used for generated keychips) if re.fullmatch(r"^A[0-9]{2}[A-Z][-]?[0-9]{2}[A-HJ-NP-Z][0-9]{4}([0-9]{4})?$", serial) is None: return False From 8f4c08f825910f39009addcfca3f72ab066df08d Mon Sep 17 00:00:00 2001 From: EmmyHeart Date: Sat, 14 Sep 2024 01:28:35 +0000 Subject: [PATCH 010/130] Fix map overload in Chusan --- titles/chuni/new.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/chuni/new.py b/titles/chuni/new.py index 2275a6e..3d3fb98 100644 --- a/titles/chuni/new.py +++ b/titles/chuni/new.py @@ -104,7 +104,8 @@ class ChuniNew(ChuniBase): return {"returnCode": "1"} async def handle_get_user_map_area_api_request(self, data: Dict) -> Dict: - user_map_areas = await self.data.item.get_map_areas(data["userId"]) + map_area_ids = [int(area["mapAreaId"]) for area in data["mapAreaIdList"]] + user_map_areas = await self.data.item.get_map_areas(data["userId"], map_area_ids) map_areas = [] for map_area in user_map_areas: From 82004cb7432965430c084fe4838d58a87181a134 Mon Sep 17 00:00:00 2001 From: EmmyHeart Date: Sat, 14 Sep 2024 01:30:29 +0000 Subject: [PATCH 011/130] Fix map overload in Chusan --- titles/chuni/schema/item.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/titles/chuni/schema/item.py b/titles/chuni/schema/item.py index 30db4b8..9ce2c53 100644 --- a/titles/chuni/schema/item.py +++ b/titles/chuni/schema/item.py @@ -533,8 +533,8 @@ class ChuniItemData(BaseData): return None return result.lastrowid - async def get_map_areas(self, user_id: int) -> Optional[List[Row]]: - sql = select(map_area).where(map_area.c.user == user_id) + async def get_map_areas(self, user_id: int, map_area_ids: List[int]) -> Optional[List[Row]]: + sql = select(map_area).where(map_area.c.user == user_id, map_area.c.mapAreaId.in_(map_area_ids)) result = await self.execute(sql) if result is None: From ee4eddd63962993e17d4efe89cc7bf038478a5b3 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 19:22:39 +0000 Subject: [PATCH 012/130] add buddies plus support --- titles/mai2/buddiesplus.py | 36 ++++++++++++++++++++++++++++++++++++ titles/mai2/const.py | 4 +++- titles/mai2/index.py | 8 ++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 titles/mai2/buddiesplus.py diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py new file mode 100644 index 0000000..0467b03 --- /dev/null +++ b/titles/mai2/buddiesplus.py @@ -0,0 +1,36 @@ +from typing import Dict + +from core.config import CoreConfig +from titles.mai2.buddies import Mai2Buddies +from titles.mai2.const import Mai2Constants +from titles.mai2.config import Mai2Config + + +class Mai2BuddiesPlus(Mai2Buddies): + def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None: + super().__init__(cfg, game_cfg) + self.version = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS + + async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: + user_data = await super().handle_cm_get_user_preview_api_request(data) + + # hardcode lastDataVersion for CardMaker + user_data["lastDataVersion"] = "1.45.00" + return user_data + + async def handle_get_game_weekly_data_api_request(self, data: Dict) -> Dict: + return { + "gameWeeklyData": { + "missionCategory": 0, + "updateDate": "2024-03-21 09:00:00.0", + "beforeDate": "2099-12-31 00:00:00.0" + } + } + + async def handle_create_token_api_request(self, data: Dict) -> Dict: + return { + "Bearer": "ARTEMiSTOKEN" # duplicate of handle_user_login_api_request from Mai2Festival + } + + async def handle_remove_token_api_request(self, data: Dict) -> Dict: + return {} diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 4dc10ce..0d13a0d 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -55,6 +55,7 @@ class Mai2Constants: VER_MAIMAI_DX_FESTIVAL = 19 VER_MAIMAI_DX_FESTIVAL_PLUS = 20 VER_MAIMAI_DX_BUDDIES = 21 + VER_MAIMAI_DX_BUDDIES_PLUS = 22 VERSION_STRING = ( "maimai", @@ -78,7 +79,8 @@ class Mai2Constants: "maimai DX UNiVERSE PLUS", "maimai DX FESTiVAL", "maimai DX FESTiVAL PLUS", - "maimai DX BUDDiES" + "maimai DX BUDDiES", + "maimai DX BUDDiES PLUS" ) @classmethod diff --git a/titles/mai2/index.py b/titles/mai2/index.py index ad02648..e8b88ec 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -30,6 +30,7 @@ from .universeplus import Mai2UniversePlus from .festival import Mai2Festival from .festivalplus import Mai2FestivalPlus from .buddies import Mai2Buddies +from .buddiesplus import Mai2BuddiesPlus class Mai2Servlet(BaseServlet): @@ -64,7 +65,8 @@ class Mai2Servlet(BaseServlet): Mai2UniversePlus, Mai2Festival, Mai2FestivalPlus, - Mai2Buddies + Mai2Buddies, + Mai2BuddiesPlus ] self.logger = logging.getLogger("mai2") @@ -302,8 +304,10 @@ class Mai2Servlet(BaseServlet): internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL elif version >= 135 and version < 140: # FESTiVAL PLUS internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS - elif version >= 140: # BUDDiES + elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES + elif version >= 145: # BUDDiES PLUS + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS elif game_code == "SDGA": # Int if version < 105: # 1.0 internal_ver = Mai2Constants.VER_MAIMAI_DX From cc302b6e56e5eb43b761ccc6a5a6a7f5b7de7178 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 19:22:47 +0000 Subject: [PATCH 013/130] update cm reader --- titles/cm/read.py | 1 + 1 file changed, 1 insertion(+) diff --git a/titles/cm/read.py b/titles/cm/read.py index 2b5ec8a..b4b3b5e 100644 --- a/titles/cm/read.py +++ b/titles/cm/read.py @@ -207,6 +207,7 @@ class CardMakerReader(BaseReader): "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL, "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS, "1.40": Mai2Constants.VER_MAIMAI_DX_BUDDIES, + "1.45": Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS } for root, dirs, files in os.walk(base_dir): From 01dad267b9962ee87edabdd2bb91b5a518e46445 Mon Sep 17 00:00:00 2001 From: ppc Date: Sun, 15 Sep 2024 20:48:24 +0000 Subject: [PATCH 014/130] add mai2 database upgrade --- .../28443e2da5b8_mai2_buddies_plus.py | 28 +++++++++++++++++++ titles/mai2/schema/profile.py | 5 +++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py diff --git a/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py new file mode 100644 index 0000000..42fcdde --- /dev/null +++ b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py @@ -0,0 +1,28 @@ +"""mai2_buddies_plus + +Revision ID: 28443e2da5b8 +Revises: 5ea73f89d982 +Create Date: 2024-09-15 20:44:02.351819 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '28443e2da5b8' +down_revision = '5ea73f89d982' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_profile_detail', sa.Column('point', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('totalPoint', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('friendRegistSkip', sa.SmallInteger())) + + +def downgrade(): + op.drop_column('mai2_profile_detail', 'point') + op.drop_column('mai2_profile_detail', 'totalPoint') + op.drop_column('mai2_profile_detail', 'friendRegistSkip') diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index c191a1a..988dcb0 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -3,7 +3,7 @@ from titles.mai2.const import Mai2Constants from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -43,6 +43,8 @@ detail = Table( Column("currentPlayCount", Integer), # new with buddies Column("renameCredit", Integer), # new with buddies Column("mapStock", Integer), # new with fes+ + Column("point", Integer), # new with buddies+ + Column("totalPoint", Integer), # new with buddies+ Column("eventWatchedDate", String(25)), Column("lastGameId", String(25)), Column("lastRomVersion", String(25)), @@ -97,6 +99,7 @@ detail = Table( Column("playerOldRating", BigInteger), Column("playerNewRating", BigInteger), Column("dateTime", BigInteger), + Column("friendRegistSkip", SmallInteger), # new with buddies+ Column("banState", Integer), # new with uni+ UniqueConstraint("user", "version", name="mai2_profile_detail_uk"), mysql_charset="utf8mb4", From b01ac24799cf98cd0cebd2184f8cf1b3329d598b Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 09:54:39 +0000 Subject: [PATCH 015/130] add shop stock/friend bonus handlers --- titles/mai2/buddiesplus.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py index 0467b03..a3784d5 100644 --- a/titles/mai2/buddiesplus.py +++ b/titles/mai2/buddiesplus.py @@ -22,15 +22,28 @@ class Mai2BuddiesPlus(Mai2Buddies): return { "gameWeeklyData": { "missionCategory": 0, - "updateDate": "2024-03-21 09:00:00.0", - "beforeDate": "2099-12-31 00:00:00.0" + "updateDate": "2024-03-21 09:00:00", + "beforeDate": "2099-12-31 00:00:00" } } - + async def handle_create_token_api_request(self, data: Dict) -> Dict: return { "Bearer": "ARTEMiSTOKEN" # duplicate of handle_user_login_api_request from Mai2Festival } - + async def handle_remove_token_api_request(self, data: Dict) -> Dict: return {} + + async def handle_get_user_friend_bonus_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "returnCode": 1, + "getMiles": 0 + } + + async def handle_get_user_shop_stock_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userShopStockList": [] + } From e128631e8fe83f0d0a14568d3ad57db27ec8b9e9 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 12:39:15 +0100 Subject: [PATCH 016/130] fix upsert failures --- titles/mai2/dx.py | 3 ++- titles/mai2/schema/item.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 7a067d7..53abc61 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -242,7 +242,8 @@ class Mai2DX(Mai2Base): if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0: for fav in upsert["userFavoriteList"]: - await self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"]) + kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ + await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) if ( "userFriendSeasonRankingList" in upsert diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index d53ebbc..13afbe4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -453,10 +453,10 @@ class Mai2ItemData(BaseData): self, user_id: int, kind: int, item_id_list: List[int] ) -> Optional[int]: sql = insert(favorite).values( - user=user_id, kind=kind, item_id_list=item_id_list + user=user_id, itemKind=kind, itemIdList=item_id_list ) - conflict = sql.on_duplicate_key_update(item_id_list=item_id_list) + conflict = sql.on_duplicate_key_update(itemIdList=item_id_list) result = await self.execute(conflict) if result is None: From 77aa1afaa0f14883ddce3c5f4243b4bce1e94b04 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 16:55:09 +0100 Subject: [PATCH 017/130] add mai2 favorite music support --- ...c91c1206dca_mai2_favorite_song_ordering.py | 24 +++++++++++++++++++ titles/mai2/base.py | 2 +- titles/mai2/dx.py | 4 ++++ titles/mai2/schema/item.py | 8 ++++--- 4 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py diff --git a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py new file mode 100644 index 0000000..ff4a62f --- /dev/null +++ b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py @@ -0,0 +1,24 @@ +"""mai2_favorite_song_ordering + +Revision ID: bc91c1206dca +Revises: 28443e2da5b8 +Create Date: 2024-09-16 14:24:56.714066 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'bc91c1206dca' +down_revision = '28443e2da5b8' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(nullable=True))) + + +def downgrade(): + op.drop_column('mai2_item_favorite_music', 'orderId') diff --git a/titles/mai2/base.py b/titles/mai2/base.py index bbd074f..b041028 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -922,7 +922,7 @@ class Mai2Base: fav_music = await self.data.item.get_fav_music(user_id) if fav_music: for fav in fav_music: - id_list.append({"orderId": 0, "id": fav["musicId"]}) + id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]}) if len(id_list) >= 100: # Lazy but whatever break diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 53abc61..0e86acd 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -245,6 +245,10 @@ class Mai2DX(Mai2Base): kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) + if "userFavoritemusicList" in upsert and len(upsert["userFavoritemusicList"]) > 0: + for fav in upsert["userFavoritemusicList"]: + await self.data.item.add_fav_music(user_id, fav["id"], fav["orderId"]) + if ( "userFriendSeasonRankingList" in upsert and len(upsert["userFriendSeasonRankingList"]) > 0 diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index 13afbe4..87ddca4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -144,6 +144,7 @@ fav_music = Table( nullable=False, ), Column("musicId", Integer, nullable=False), + Column("orderId", Integer, nullable=True), UniqueConstraint("user", "musicId", name="mai2_item_favorite_music_uk"), mysql_charset="utf8mb4", ) @@ -484,13 +485,14 @@ class Mai2ItemData(BaseData): if result: return result.fetchall() - async def add_fav_music(self, user_id: int, music_id: int) -> Optional[int]: + async def add_fav_music(self, user_id: int, music_id: int, order_id: Optional[int] = None) -> Optional[int]: sql = insert(fav_music).values( user = user_id, - musicId = music_id + musicId = music_id, + orderId = order_id ) - conflict = sql.on_duplicate_key_update(musicId = music_id) + conflict = sql.on_duplicate_key_update(orderId = order_id) result = await self.execute(conflict) if result: From d8169e37cc88b2041b4a4fec2151486037459ae6 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 17:56:22 +0100 Subject: [PATCH 018/130] add mai2 UserIntimateApi --- .../versions/54a84103b84e_mai2_intimacy.py | 43 +++++++++++++++++++ titles/mai2/dx.py | 23 ++++++++++ titles/mai2/schema/profile.py | 37 ++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 core/data/alembic/versions/54a84103b84e_mai2_intimacy.py diff --git a/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py new file mode 100644 index 0000000..a180bbb --- /dev/null +++ b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py @@ -0,0 +1,43 @@ +"""mai2_intimacy + +Revision ID: 54a84103b84e +Revises: bc91c1206dca +Create Date: 2024-09-16 17:47:49.164546 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import Column, Integer, UniqueConstraint + +# revision identifiers, used by Alembic. +revision = '54a84103b84e' +down_revision = 'bc91c1206dca' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "mai2_user_intimate", + Column("id", Integer, primary_key=True, nullable=False), + Column("user", Integer, nullable=False), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", + ) + + op.create_foreign_key( + None, + "mai2_user_intimate", + "aime_user", + ["user"], + ["id"], + ondelete="cascade", + onupdate="cascade", + ) + + +def downgrade(): + op.drop_table("mai2_user_intimate") diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 0e86acd..a31f98c 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -264,6 +264,11 @@ class Mai2DX(Mai2Base): if "user2pPlaylog" in upsert: await self.data.score.put_playlog_2p(user_id, upsert["user2pPlaylog"]) + # added in BUDDiES+ + if "userIntimateList" in upsert and len(upsert["userIntimateList"]) > 0: + for intimate in upsert["userIntimateList"]: + await self.data.profile.put_intimacy(user_id, intimate["partnerId"], intimate["intimateLevel"], intimate["intimateCountRewarded"]) + return {"returnCode": 1, "apiName": "UpsertUserAllApi"} async def handle_get_user_data_api_request(self, data: Dict) -> Dict: @@ -713,6 +718,24 @@ class Mai2DX(Mai2Base): ret['loginId'] = ret.get('loginCount', 0) return ret + # Intimate api added in BUDDiES+ + async def handle_get_user_intimate_api_request(self, data: Dict) -> Dict: + intimate = await self.data.profile.get_intimacy(data["userId"]) + if intimate is None: + return {} + + partner_list = [{ + "partnerId": i["partnerId"], + "intimateLevel": i["intimateLevel"], + "intimateCountRewarded": i["intimateCountRewarded"] + } for i in intimate] + + return { + "userId": data["userId"], + "length": len(partner_list), + "userIntimateList": partner_list + } + # CardMaker support added in Universe async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: p = await self.data.profile.get_profile_detail(data["userId"], self.version) diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index 988dcb0..3ff85d2 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -513,6 +513,22 @@ rival = Table( mysql_charset="utf8mb4", ) +intimacy = Table( +"mai2_user_intimate", + metadata, + Column("id", Integer, primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -908,6 +924,27 @@ class Mai2ProfileData(BaseData): if not result: self.logger.error(f"Failed to remove rival {rival_id} for user {user_id}!") + async def get_intimacy(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(intimacy.select(intimacy.c.user == user_id)) + if result: + return result.fetchall() + + async def put_intimacy(self, user_id: int, partner_id: int, level: int, count_rewarded: int) -> Optional[int]: + sql = insert(intimacy).values( + user = user_id, + partnerId = partner_id, + intimateLevel = level, + intimateCountRewarded = count_rewarded + ) + + conflict = sql.on_duplicate_key_update(intimateLevel = level, intimateCountRewarded = count_rewarded) + + result = await self.execute(conflict) + if result: + return result.lastrowid + + self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name From 196aa601f3e461329af095b0e3b5120d30e858c3 Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 16 Sep 2024 18:01:06 +0100 Subject: [PATCH 019/130] handle GetUserMissionDataApi --- titles/mai2/buddiesplus.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py index a3784d5..e87fae6 100644 --- a/titles/mai2/buddiesplus.py +++ b/titles/mai2/buddiesplus.py @@ -47,3 +47,14 @@ class Mai2BuddiesPlus(Mai2Buddies): "userId": data["userId"], "userShopStockList": [] } + + async def handle_get_user_mission_data_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userMissionDataList": [], + "userWeeklyData": { + "lastLoginWeek": "2024-03-21 09:00:00", + "beforeLoginWeek": "2099-12-31 00:00:00", + "friendBonusFlag": False + } + } From d85c575c61006e340722e69eae217f38a8d2b3e7 Mon Sep 17 00:00:00 2001 From: ppc Date: Wed, 18 Sep 2024 11:29:14 +0100 Subject: [PATCH 020/130] add nullcheck --- titles/mai2/dx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index a31f98c..66bf914 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -243,7 +243,8 @@ class Mai2DX(Mai2Base): if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0: for fav in upsert["userFavoriteList"]: kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ - await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) + if kind_id is not None: + await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) if "userFavoritemusicList" in upsert and len(upsert["userFavoritemusicList"]) > 0: for fav in upsert["userFavoritemusicList"]: From 8d04d74f523faca71b3b7552124c9f660aa041e9 Mon Sep 17 00:00:00 2001 From: ppc Date: Wed, 18 Sep 2024 17:59:24 +0100 Subject: [PATCH 021/130] migration consistency --- .../versions/bc91c1206dca_mai2_favorite_song_ordering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py index ff4a62f..abf6357 100644 --- a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py +++ b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py @@ -17,7 +17,7 @@ depends_on = None def upgrade(): - op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(nullable=True))) + op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(), nullable=True)) def downgrade(): From 5c60cde14f89b9db2212f8963bcec20ade88c75e Mon Sep 17 00:00:00 2001 From: ppc Date: Thu, 19 Sep 2024 23:10:54 +0100 Subject: [PATCH 022/130] update docs --- docs/game_specific_info.md | 1 + readme.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 37561fc..a0aed71 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -218,6 +218,7 @@ Presents are items given to the user when they login, with a little animation (f | SDEZ | 19 | maimai DX FESTiVAL | | SDEZ | 20 | maimai DX FESTiVAL PLUS | | SDEZ | 21 | maimai DX BUDDiES | +| SDEZ | 22 | maimai DX BUDDiES PLUS | ### Importer diff --git a/readme.md b/readme.md index c792928..0564c70 100644 --- a/readme.md +++ b/readme.md @@ -50,6 +50,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + FESTiVAL + FESTiVAL PLUS + BUDDiES + + BUDDiES PLUS + O.N.G.E.K.I. + SUMMER From e85728f33cb9deb297ef599ce9a8799e1cf0836c Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Fri, 20 Sep 2024 17:10:48 -0400 Subject: [PATCH 023/130] chuni/mai2: remove upsert from put_playlog --- titles/chuni/schema/score.py | 3 +-- titles/mai2/schema/score.py | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/titles/chuni/schema/score.py b/titles/chuni/schema/score.py index 766b4b9..0d327f8 100644 --- a/titles/chuni/schema/score.py +++ b/titles/chuni/schema/score.py @@ -231,9 +231,8 @@ class ChuniScoreData(BaseData): playlog_data["romVersion"] = romVer.get(version, "1.00.0") sql = insert(playlog).values(**playlog_data) - conflict = sql.on_duplicate_key_update(**playlog_data) - result = await self.execute(conflict) + result = await self.execute(sql) if result is None: return None return result.lastrowid diff --git a/titles/mai2/schema/score.py b/titles/mai2/schema/score.py index d4ea5b9..f62466a 100644 --- a/titles/mai2/schema/score.py +++ b/titles/mai2/schema/score.py @@ -359,9 +359,7 @@ class Mai2ScoreData(BaseData): else: sql = insert(playlog_old).values(**playlog_data) - conflict = sql.on_duplicate_key_update(**playlog_data) - - result = await self.execute(conflict) + result = await self.execute(sql) if result is None: self.logger.error(f"put_playlog: Failed to insert! user_id {user_id} is_dx {is_dx}") return None @@ -371,9 +369,7 @@ class Mai2ScoreData(BaseData): playlog_2p_data["user"] = user_id sql = insert(playlog_2p).values(**playlog_2p_data) - conflict = sql.on_duplicate_key_update(**playlog_2p_data) - - result = await self.execute(conflict) + result = await self.execute(sql) if result is None: self.logger.error(f"put_playlog_2p: Failed to insert! user_id {user_id}") return None From f47175a1440bc71cad51b2cd56fcda0ff4afb89d Mon Sep 17 00:00:00 2001 From: ppc Date: Mon, 23 Sep 2024 17:21:29 +0000 Subject: [PATCH 024/130] [mai2] add buddies plus support (#177) Adds favorite music support (there's an option in the results screen to star a song), handlers for new methods and fixes upsert failures for `userFavoriteList`. The `UserIntimateApi` has been added but didn't seem to add any data during testing, and `CreateTokenApi`/`RemoveTokenApi` have also been added but I think they're only used during guest play. --- Tested on 1.45 with no errors/game crashes (see logs). Card Maker hasn't been tested as I don't have a setup to play with. Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/177 Co-authored-by: ppc Co-committed-by: ppc --- .../28443e2da5b8_mai2_buddies_plus.py | 28 +++++++++ .../versions/54a84103b84e_mai2_intimacy.py | 43 +++++++++++++ ...c91c1206dca_mai2_favorite_song_ordering.py | 24 ++++++++ docs/game_specific_info.md | 1 + readme.md | 1 + titles/cm/read.py | 1 + titles/mai2/base.py | 2 +- titles/mai2/buddiesplus.py | 60 +++++++++++++++++++ titles/mai2/const.py | 4 +- titles/mai2/dx.py | 31 +++++++++- titles/mai2/index.py | 8 ++- titles/mai2/schema/item.py | 12 ++-- titles/mai2/schema/profile.py | 42 ++++++++++++- 13 files changed, 246 insertions(+), 11 deletions(-) create mode 100644 core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py create mode 100644 core/data/alembic/versions/54a84103b84e_mai2_intimacy.py create mode 100644 core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py create mode 100644 titles/mai2/buddiesplus.py diff --git a/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py new file mode 100644 index 0000000..42fcdde --- /dev/null +++ b/core/data/alembic/versions/28443e2da5b8_mai2_buddies_plus.py @@ -0,0 +1,28 @@ +"""mai2_buddies_plus + +Revision ID: 28443e2da5b8 +Revises: 5ea73f89d982 +Create Date: 2024-09-15 20:44:02.351819 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '28443e2da5b8' +down_revision = '5ea73f89d982' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_profile_detail', sa.Column('point', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('totalPoint', sa.Integer())) + op.add_column('mai2_profile_detail', sa.Column('friendRegistSkip', sa.SmallInteger())) + + +def downgrade(): + op.drop_column('mai2_profile_detail', 'point') + op.drop_column('mai2_profile_detail', 'totalPoint') + op.drop_column('mai2_profile_detail', 'friendRegistSkip') diff --git a/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py new file mode 100644 index 0000000..a180bbb --- /dev/null +++ b/core/data/alembic/versions/54a84103b84e_mai2_intimacy.py @@ -0,0 +1,43 @@ +"""mai2_intimacy + +Revision ID: 54a84103b84e +Revises: bc91c1206dca +Create Date: 2024-09-16 17:47:49.164546 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import Column, Integer, UniqueConstraint + +# revision identifiers, used by Alembic. +revision = '54a84103b84e' +down_revision = 'bc91c1206dca' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "mai2_user_intimate", + Column("id", Integer, primary_key=True, nullable=False), + Column("user", Integer, nullable=False), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", + ) + + op.create_foreign_key( + None, + "mai2_user_intimate", + "aime_user", + ["user"], + ["id"], + ondelete="cascade", + onupdate="cascade", + ) + + +def downgrade(): + op.drop_table("mai2_user_intimate") diff --git a/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py new file mode 100644 index 0000000..abf6357 --- /dev/null +++ b/core/data/alembic/versions/bc91c1206dca_mai2_favorite_song_ordering.py @@ -0,0 +1,24 @@ +"""mai2_favorite_song_ordering + +Revision ID: bc91c1206dca +Revises: 28443e2da5b8 +Create Date: 2024-09-16 14:24:56.714066 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'bc91c1206dca' +down_revision = '28443e2da5b8' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('mai2_item_favorite_music', sa.Column('orderId', sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column('mai2_item_favorite_music', 'orderId') diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 37561fc..a0aed71 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -218,6 +218,7 @@ Presents are items given to the user when they login, with a little animation (f | SDEZ | 19 | maimai DX FESTiVAL | | SDEZ | 20 | maimai DX FESTiVAL PLUS | | SDEZ | 21 | maimai DX BUDDiES | +| SDEZ | 22 | maimai DX BUDDiES PLUS | ### Importer diff --git a/readme.md b/readme.md index c792928..0564c70 100644 --- a/readme.md +++ b/readme.md @@ -50,6 +50,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + FESTiVAL + FESTiVAL PLUS + BUDDiES + + BUDDiES PLUS + O.N.G.E.K.I. + SUMMER diff --git a/titles/cm/read.py b/titles/cm/read.py index 2b5ec8a..b4b3b5e 100644 --- a/titles/cm/read.py +++ b/titles/cm/read.py @@ -207,6 +207,7 @@ class CardMakerReader(BaseReader): "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL, "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS, "1.40": Mai2Constants.VER_MAIMAI_DX_BUDDIES, + "1.45": Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS } for root, dirs, files in os.walk(base_dir): diff --git a/titles/mai2/base.py b/titles/mai2/base.py index bbd074f..b041028 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -922,7 +922,7 @@ class Mai2Base: fav_music = await self.data.item.get_fav_music(user_id) if fav_music: for fav in fav_music: - id_list.append({"orderId": 0, "id": fav["musicId"]}) + id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]}) if len(id_list) >= 100: # Lazy but whatever break diff --git a/titles/mai2/buddiesplus.py b/titles/mai2/buddiesplus.py new file mode 100644 index 0000000..e87fae6 --- /dev/null +++ b/titles/mai2/buddiesplus.py @@ -0,0 +1,60 @@ +from typing import Dict + +from core.config import CoreConfig +from titles.mai2.buddies import Mai2Buddies +from titles.mai2.const import Mai2Constants +from titles.mai2.config import Mai2Config + + +class Mai2BuddiesPlus(Mai2Buddies): + def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None: + super().__init__(cfg, game_cfg) + self.version = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS + + async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: + user_data = await super().handle_cm_get_user_preview_api_request(data) + + # hardcode lastDataVersion for CardMaker + user_data["lastDataVersion"] = "1.45.00" + return user_data + + async def handle_get_game_weekly_data_api_request(self, data: Dict) -> Dict: + return { + "gameWeeklyData": { + "missionCategory": 0, + "updateDate": "2024-03-21 09:00:00", + "beforeDate": "2099-12-31 00:00:00" + } + } + + async def handle_create_token_api_request(self, data: Dict) -> Dict: + return { + "Bearer": "ARTEMiSTOKEN" # duplicate of handle_user_login_api_request from Mai2Festival + } + + async def handle_remove_token_api_request(self, data: Dict) -> Dict: + return {} + + async def handle_get_user_friend_bonus_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "returnCode": 1, + "getMiles": 0 + } + + async def handle_get_user_shop_stock_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userShopStockList": [] + } + + async def handle_get_user_mission_data_api_request(self, data: Dict) -> Dict: + return { + "userId": data["userId"], + "userMissionDataList": [], + "userWeeklyData": { + "lastLoginWeek": "2024-03-21 09:00:00", + "beforeLoginWeek": "2099-12-31 00:00:00", + "friendBonusFlag": False + } + } diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 4dc10ce..0d13a0d 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -55,6 +55,7 @@ class Mai2Constants: VER_MAIMAI_DX_FESTIVAL = 19 VER_MAIMAI_DX_FESTIVAL_PLUS = 20 VER_MAIMAI_DX_BUDDIES = 21 + VER_MAIMAI_DX_BUDDIES_PLUS = 22 VERSION_STRING = ( "maimai", @@ -78,7 +79,8 @@ class Mai2Constants: "maimai DX UNiVERSE PLUS", "maimai DX FESTiVAL", "maimai DX FESTiVAL PLUS", - "maimai DX BUDDiES" + "maimai DX BUDDiES", + "maimai DX BUDDiES PLUS" ) @classmethod diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 7a067d7..66bf914 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -242,7 +242,13 @@ class Mai2DX(Mai2Base): if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0: for fav in upsert["userFavoriteList"]: - await self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"]) + kind_id = fav.get("kind", fav.get("itemKind")) # itemKind key used in BUDDiES+ + if kind_id is not None: + await self.data.item.put_favorite(user_id, kind_id, fav["itemIdList"]) + + if "userFavoritemusicList" in upsert and len(upsert["userFavoritemusicList"]) > 0: + for fav in upsert["userFavoritemusicList"]: + await self.data.item.add_fav_music(user_id, fav["id"], fav["orderId"]) if ( "userFriendSeasonRankingList" in upsert @@ -259,6 +265,11 @@ class Mai2DX(Mai2Base): if "user2pPlaylog" in upsert: await self.data.score.put_playlog_2p(user_id, upsert["user2pPlaylog"]) + # added in BUDDiES+ + if "userIntimateList" in upsert and len(upsert["userIntimateList"]) > 0: + for intimate in upsert["userIntimateList"]: + await self.data.profile.put_intimacy(user_id, intimate["partnerId"], intimate["intimateLevel"], intimate["intimateCountRewarded"]) + return {"returnCode": 1, "apiName": "UpsertUserAllApi"} async def handle_get_user_data_api_request(self, data: Dict) -> Dict: @@ -708,6 +719,24 @@ class Mai2DX(Mai2Base): ret['loginId'] = ret.get('loginCount', 0) return ret + # Intimate api added in BUDDiES+ + async def handle_get_user_intimate_api_request(self, data: Dict) -> Dict: + intimate = await self.data.profile.get_intimacy(data["userId"]) + if intimate is None: + return {} + + partner_list = [{ + "partnerId": i["partnerId"], + "intimateLevel": i["intimateLevel"], + "intimateCountRewarded": i["intimateCountRewarded"] + } for i in intimate] + + return { + "userId": data["userId"], + "length": len(partner_list), + "userIntimateList": partner_list + } + # CardMaker support added in Universe async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: p = await self.data.profile.get_profile_detail(data["userId"], self.version) diff --git a/titles/mai2/index.py b/titles/mai2/index.py index ad02648..e8b88ec 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -30,6 +30,7 @@ from .universeplus import Mai2UniversePlus from .festival import Mai2Festival from .festivalplus import Mai2FestivalPlus from .buddies import Mai2Buddies +from .buddiesplus import Mai2BuddiesPlus class Mai2Servlet(BaseServlet): @@ -64,7 +65,8 @@ class Mai2Servlet(BaseServlet): Mai2UniversePlus, Mai2Festival, Mai2FestivalPlus, - Mai2Buddies + Mai2Buddies, + Mai2BuddiesPlus ] self.logger = logging.getLogger("mai2") @@ -302,8 +304,10 @@ class Mai2Servlet(BaseServlet): internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL elif version >= 135 and version < 140: # FESTiVAL PLUS internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS - elif version >= 140: # BUDDiES + elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES + elif version >= 145: # BUDDiES PLUS + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS elif game_code == "SDGA": # Int if version < 105: # 1.0 internal_ver = Mai2Constants.VER_MAIMAI_DX diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index d53ebbc..87ddca4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -144,6 +144,7 @@ fav_music = Table( nullable=False, ), Column("musicId", Integer, nullable=False), + Column("orderId", Integer, nullable=True), UniqueConstraint("user", "musicId", name="mai2_item_favorite_music_uk"), mysql_charset="utf8mb4", ) @@ -453,10 +454,10 @@ class Mai2ItemData(BaseData): self, user_id: int, kind: int, item_id_list: List[int] ) -> Optional[int]: sql = insert(favorite).values( - user=user_id, kind=kind, item_id_list=item_id_list + user=user_id, itemKind=kind, itemIdList=item_id_list ) - conflict = sql.on_duplicate_key_update(item_id_list=item_id_list) + conflict = sql.on_duplicate_key_update(itemIdList=item_id_list) result = await self.execute(conflict) if result is None: @@ -484,13 +485,14 @@ class Mai2ItemData(BaseData): if result: return result.fetchall() - async def add_fav_music(self, user_id: int, music_id: int) -> Optional[int]: + async def add_fav_music(self, user_id: int, music_id: int, order_id: Optional[int] = None) -> Optional[int]: sql = insert(fav_music).values( user = user_id, - musicId = music_id + musicId = music_id, + orderId = order_id ) - conflict = sql.on_duplicate_key_update(musicId = music_id) + conflict = sql.on_duplicate_key_update(orderId = order_id) result = await self.execute(conflict) if result: diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index c191a1a..3ff85d2 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -3,7 +3,7 @@ from titles.mai2.const import Mai2Constants from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -43,6 +43,8 @@ detail = Table( Column("currentPlayCount", Integer), # new with buddies Column("renameCredit", Integer), # new with buddies Column("mapStock", Integer), # new with fes+ + Column("point", Integer), # new with buddies+ + Column("totalPoint", Integer), # new with buddies+ Column("eventWatchedDate", String(25)), Column("lastGameId", String(25)), Column("lastRomVersion", String(25)), @@ -97,6 +99,7 @@ detail = Table( Column("playerOldRating", BigInteger), Column("playerNewRating", BigInteger), Column("dateTime", BigInteger), + Column("friendRegistSkip", SmallInteger), # new with buddies+ Column("banState", Integer), # new with uni+ UniqueConstraint("user", "version", name="mai2_profile_detail_uk"), mysql_charset="utf8mb4", @@ -510,6 +513,22 @@ rival = Table( mysql_charset="utf8mb4", ) +intimacy = Table( +"mai2_user_intimate", + metadata, + Column("id", Integer, primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("partnerId", Integer, nullable=False), + Column("intimateLevel", Integer, nullable=False), + Column("intimateCountRewarded", Integer, nullable=False), + UniqueConstraint("user", "partnerId", name="mai2_user_intimate_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -905,6 +924,27 @@ class Mai2ProfileData(BaseData): if not result: self.logger.error(f"Failed to remove rival {rival_id} for user {user_id}!") + async def get_intimacy(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(intimacy.select(intimacy.c.user == user_id)) + if result: + return result.fetchall() + + async def put_intimacy(self, user_id: int, partner_id: int, level: int, count_rewarded: int) -> Optional[int]: + sql = insert(intimacy).values( + user = user_id, + partnerId = partner_id, + intimateLevel = level, + intimateCountRewarded = count_rewarded + ) + + conflict = sql.on_duplicate_key_update(intimateLevel = level, intimateCountRewarded = count_rewarded) + + result = await self.execute(conflict) + if result: + return result.lastrowid + + self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name From aa8e33a13ee33082a139cd65a6c40c4513033a95 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Mon, 23 Sep 2024 14:20:25 -0400 Subject: [PATCH 025/130] docs: add pokken to game specific info --- docs/game_specific_info.md | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index a0aed71..6c938d7 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -31,6 +31,7 @@ python dbutils.py migrate - [WACCA](#wacca) - [Sword Art Online Arcade](#sao) - [Initial D THE ARCADE](#initial-d-the-arcade) + - [Pokken Tournament](#pokken) # Supported Games @@ -797,3 +798,82 @@ python dbutils.py upgrade A huge thanks to all people who helped shaping this project to what it is now and don't want to be mentioned here. +## Pokken + +### SDAK + +| Version ID | Version Name | +| ---------- | ------------ | +| 0 | Pokken | + +### Config + +Config file is `pokken.yaml` + +#### server + +| Option | Info | Default | +| ------ | ---- | ------- | +| `hostname` | Hostname override for allnet to tell the game where to connect. Useful for local setups that need to use a different hostname for pokken's proxy. Otherwise, it should match `server`->`hostname` in `core.yaml`. | `localhost` | +| `enabled` | `True` if the pokken service should be enabled. `False` otherwise. | `True` | +| `loglevel` | String indicating how verbose pokken logs should be. Acceptable values are `debug`, `info`, `warn`, and `error`. | `info` | +| `auto_register` | For games that don't use aimedb, this controls weather connecting cards that aren't registered should automatically be registered when making a profile. Set to `False` to require cards be already registered before being usable with Pokken. | `True` | +| `enable_matching` | If `True`, allow non-local matching. This doesn't currently work because BIWA, the matching protocol the game uses, is not understood, so this should be set to `False`. | `False` | +| `stun_server_host` | Hostname of the STUN server the game will use for matching. | `stunserver.stunprotocol.org` (might not work anymore? recomend changing) | +| `stun_server_port` | Port for the external STUN server. Will probably be moved to the `ports` section in the future. | `3478` | + +#### ports +| Option | Info | Default | +| ------ | ---- | ------- | +| `game` | Override for the title server port sent by allnet. Useful for local setups utalizing NGINX. | `9000` | +| `admission` | Port for the admission server used in global matching. May be obsolited later. | `9001` | + +### Connecting to Artemis + +Pokken is a bit tricky to get working due to it having a hard requirement of the connection being HTTPS. This is simplified somewhat by Pokken simply not validating the certificate in any way, shape or form (it can be self-signed, expired, for a different domain, etc.) but it does have to be there. The work-around is to spin up a local NGINX (or other proxy) instance and point traffic back to artemis. See below for a sample nginx config: +`nginx.conf` +```conf +# This example assumes your artemis instance is configured to listed on port 8080, and your certs exists at /path/to/cert and are called title.crt and title.key. +server { + listen 443 ssl; + server_name your.hostname.here; + + ssl_certificate /path/to/cert/title.crt; + ssl_certificate_key /path/to/cert/title.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; + ssl_session_tickets off; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers "ALL:@SECLEVEL=0"; + ssl_prefer_server_ciphers off; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass_request_headers on; + proxy_pass http://127.0.0.1:8080/; + } +} +``` +`pokken.yaml` +```yaml +server: + hostname: "your.hostname.here" + enable: True + loglevel: "info" + auto_register: True + enable_matching: False + stun_server_host: "stunserver.stunprotocol.org" + stun_server_port: 3478 + +ports: + game: 443 + admission: 9001 +``` + +### Info + +The arcade release is missing a few fighters and supports compared to the switch version. It may be possible to mod these in in the future, but not much headway has been made on this as far as I know. Mercifully, the game uses the pokedex number (illustration_book_no) wherever possible when referingto both fighters and supports. Customization is entirely done on the webui. Artemis currently only supports changing your name, gender, and supporrt teams, but more is planned for the future. + +### Credits +Special thanks to Pocky for pointing me in the right direction in terms of getting this game to function at all, and Lightning and other pokken cab owners for doing testing and reporting bugs/issues. From 045465ed4ed9e18cbf3bd4247d2fe3a6552ec675 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Mon, 23 Sep 2024 14:46:41 -0400 Subject: [PATCH 026/130] idz: disabled by default to silence warnings for people who don't feel like configuring games they don't intend to use --- example_config/idz.yaml | 2 +- titles/idz/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example_config/idz.yaml b/example_config/idz.yaml index 1bfff9b..3ec39b1 100644 --- a/example_config/idz.yaml +++ b/example_config/idz.yaml @@ -1,5 +1,5 @@ server: - enable: True + enable: False loglevel: "info" hostname: "" news: "" diff --git a/titles/idz/config.py b/titles/idz/config.py index f7af4fd..3c8e870 100644 --- a/titles/idz/config.py +++ b/titles/idz/config.py @@ -10,7 +10,7 @@ class IDZServerConfig: @property def enable(self) -> bool: return CoreConfig.get_config_field( - self.__config, "idz", "server", "enable", default=True + self.__config, "idz", "server", "enable", default=False ) @property From 1d8e31d4abca3149c713df8416744dfd5f51995a Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Mon, 23 Sep 2024 14:46:48 -0400 Subject: [PATCH 027/130] docs: add missing games --- docs/game_specific_info.md | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 6c938d7..4231297 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -26,10 +26,12 @@ python dbutils.py migrate - [CHUNITHM](#chunithm) - [crossbeats REV.](#crossbeats-rev) - [maimai DX](#maimai-dx) + - [Project Diva](#hatsune-miku-project-diva) - [O.N.G.E.K.I.](#o-n-g-e-k-i) - [Card Maker](#card-maker) - [WACCA](#wacca) - [Sword Art Online Arcade](#sao) + - [Initial D Zero](#initial-d-zero) - [Initial D THE ARCADE](#initial-d-the-arcade) - [Pokken Tournament](#pokken) @@ -294,6 +296,23 @@ Always make sure your database (tables) are up-to-date: python dbutils.py upgrade ``` +### Using NGINX + +Diva's netcode does not send a `Host` header with it's network requests. This renders it incompatable with NGINX as configured in the example config, because nginx relies on the header to determine how to proxy the request. If you'd still like to use NGINX with diva, please see the sample config below. + +```conf +server { + listen 80 default_server; + server_name _; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass_request_headers on; + proxy_pass http://127.0.0.1:8080/; + } +} +``` + ## O.N.G.E.K.I. ### SDDT @@ -652,21 +671,32 @@ python dbutils.py upgrade ``` ### Notes -- Defrag Match will crash at loading -- Co-Op Online is not supported -- Shop is displayed but cannot purchase heroes or items +- Defrag Match and online coop requires a cloud instance of Photon and a working application ID - Player title is currently static and cannot be changed in-game -- QR Card Scanning currently only load a static hero -- Ex-quests progression not supported yet +- QR Card Scanning of existing cards requires them to be registered on the webui - Daily Missions not implemented -- EX TOWER 1,2 & 3 are not yet supported -- Daily Yui coin not yet fixed +- Terminal functionality is almost entirely untested ### Credits for SAO support: - Midorica - Network Support - Dniel97 - Helping with network base - tungnotpunk - Source +- Hay1tsme - fixing many issues with the original implemetation + +## Initial D Zero +### SDDF + +| Version ID | Version Name | +| ---------- | -------------------- | +| 0 | Initial D Zero v1.10 | +| 1 | Initial D Zero v1.30 | +| 2 | Initial D Zero v2.10 | +| 3 | Initial D Zero v2.30 | + +### Info + +TODO, probably just leave disabled unless you're doing development things for it. ## Initial D THE ARCADE From b04840f3dd3fbb516d176a3d031ba8dc3679afec Mon Sep 17 00:00:00 2001 From: daydensteve Date: Wed, 25 Sep 2024 14:53:43 +0000 Subject: [PATCH 028/130] [chuni] Frontend favorites support (#176) I had been itching for the favorites feature since I'm bad with japanese so figured I'd go ahead and add it. I've included a few pics to help visualize the changes. ### Summary of user-facing changes: - New Favorites frontend page that itemizes favorites by genre for the current version (as selected on the Profile page). Favorites can be removed from this page via the Remove button - Updated the Records page so that it only shows the playlog for the currently selected version and includes a "star" to the left of each title that can be clicked to add/remove favorites. When the star is yellow, its a favorite; when its a grey outline, its not. I figure its pretty straight forward - The Records and new Favorites pages show the jacket image of each song now (The Importer was updated to convert the DDS files to PNGs on import) ### Behind-the-scenes changes: - Fixed a bug in the chuni get_song method - it was inappropriately comparing the row id instead of the musicid (note this method was not used prior to adding favorites support) - Overhauled the score scheme file to stop with all the hacky romVersion determination that was going on in various methods. To do this, I created a new ChuniRomVersion class that is populated with all base rom versions, then used to derive the internal integer version number from the string stored in the DB. As written, this functionality can infer recorded rom versions when the playlog was entered using an update to the base version (e.g. 2.16 vs 2.15 for sunplus or 2.22 vs 2.20 for luminous). - Made the chuni config version class safer as it would previously throw an exception if you gave it a version not present in the config file. This was done in support of the score overhaul to build up the initial ChuniRomVersion dict - Added necessary methods to query/update the favorites table. ### Testing - Frontend testing was performed with playlog data for both sunplus (2.16) and luminous (2.22) present. All add/remove permutations and images behavior was as expected - Game testing was performed only with Luminous (2.22) and worked fine Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/176 Co-authored-by: daydensteve Co-committed-by: daydensteve --- titles/chuni/config.py | 9 +- titles/chuni/database.py | 8 +- titles/chuni/frontend.py | 122 +++++++++++- titles/chuni/img/jacket/unknown.png | Bin 0 -> 27489 bytes titles/chuni/read.py | 11 ++ titles/chuni/schema/__init__.py | 4 +- titles/chuni/schema/item.py | 44 +++++ titles/chuni/schema/score.py | 186 ++++++++++++++----- titles/chuni/schema/static.py | 2 +- titles/chuni/templates/chuni_favorites.jinja | 55 ++++++ titles/chuni/templates/chuni_header.jinja | 3 + titles/chuni/templates/chuni_playlog.jinja | 23 ++- titles/chuni/templates/css/chuni_style.css | 17 ++ 13 files changed, 418 insertions(+), 66 deletions(-) create mode 100644 titles/chuni/img/jacket/unknown.png create mode 100644 titles/chuni/templates/chuni_favorites.jinja diff --git a/titles/chuni/config.py b/titles/chuni/config.py index dcdfce4..51f819c 100644 --- a/titles/chuni/config.py +++ b/titles/chuni/config.py @@ -75,9 +75,14 @@ class ChuniVersionConfig: in the form of: 11: {"rom": 2.00.00, "data": 2.00.00} """ - return CoreConfig.get_config_field( + versions = CoreConfig.get_config_field( self.__config, "chuni", "version", default={} - )[version] + ) + + if version not in versions.keys(): + return None + + return versions[version] class ChuniCryptoConfig: diff --git a/titles/chuni/database.py b/titles/chuni/database.py index eeb588c..1d5b800 100644 --- a/titles/chuni/database.py +++ b/titles/chuni/database.py @@ -1,13 +1,17 @@ from core.data import Data from core.config import CoreConfig from titles.chuni.schema import * - +from .config import ChuniConfig class ChuniData(Data): - def __init__(self, cfg: CoreConfig) -> None: + def __init__(self, cfg: CoreConfig, chuni_cfg: ChuniConfig = None) -> None: super().__init__(cfg) self.item = ChuniItemData(cfg, self.session) self.profile = ChuniProfileData(cfg, self.session) self.score = ChuniScoreData(cfg, self.session) self.static = ChuniStaticData(cfg, self.session) + + # init rom versioning for use with score playlog data + if chuni_cfg: + ChuniRomVersion.init_versions(chuni_cfg) diff --git a/titles/chuni/frontend.py b/titles/chuni/frontend.py index 74f7794..69f1ae9 100644 --- a/titles/chuni/frontend.py +++ b/titles/chuni/frontend.py @@ -2,6 +2,7 @@ from typing import List from starlette.routing import Route, Mount from starlette.requests import Request from starlette.responses import Response, RedirectResponse +from starlette.staticfiles import StaticFiles from os import path import yaml import jinja2 @@ -81,12 +82,12 @@ class ChuniFrontend(FE_Base): self, cfg: CoreConfig, environment: jinja2.Environment, cfg_dir: str ) -> None: super().__init__(cfg, environment) - self.data = ChuniData(cfg) self.game_cfg = ChuniConfig() if path.exists(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}"): self.game_cfg.update( yaml.safe_load(open(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}")) ) + self.data = ChuniData(cfg, self.game_cfg) self.nav_name = "Chunithm" def get_routes(self) -> List[Route]: @@ -97,8 +98,12 @@ class ChuniFrontend(FE_Base): Route("/", self.render_GET_playlog, methods=['GET']), Route("/{index}", self.render_GET_playlog, methods=['GET']), ]), + Route("/favorites", self.render_GET_favorites, methods=['GET']), Route("/update.name", self.update_name, methods=['POST']), + Route("/update.favorite_music_playlog", self.update_favorite_music_playlog, methods=['POST']), + Route("/update.favorite_music_favorites", self.update_favorite_music_favorites, methods=['POST']), Route("/version.change", self.version_change, methods=['POST']), + Mount('/img', app=StaticFiles(directory='titles/chuni/img'), name="img") ] async def render_GET(self, request: Request) -> bytes: @@ -205,7 +210,8 @@ class ChuniFrontend(FE_Base): else: index = int(path_index) - 1 # 0 and 1 are 1st page user_id = usr_sesh.user_id - playlog_count = await self.data.score.get_user_playlogs_count(user_id) + version = usr_sesh.chunithm_version + playlog_count = await self.data.score.get_user_playlogs_count(user_id, version) if playlog_count < index * 20 : return Response(template.render( title=f"{self.core_config.server.name} | {self.nav_name}", @@ -213,31 +219,107 @@ class ChuniFrontend(FE_Base): sesh=vars(usr_sesh), playlog_count=0 ), media_type="text/html; charset=utf-8") - playlog = await self.data.score.get_playlogs_limited(user_id, index, 20) + playlog = await self.data.score.get_playlogs_limited(user_id, version, index, 20) playlog_with_title = [] - for record in playlog: - music_chart = await self.data.static.get_music_chart(usr_sesh.chunithm_version, record.musicId, record.level) + for idx,record in enumerate(playlog): + music_chart = await self.data.static.get_music_chart(version, record.musicId, record.level) if music_chart: difficultyNum=music_chart.level artist=music_chart.artist title=music_chart.title + (jacket, ext) = path.splitext(music_chart.jacketPath) + jacket += ".png" else: difficultyNum=0 artist="unknown" title="musicid: " + str(record.musicId) + jacket = "unknown.png" + + # Check if this song is a favorite so we can populate the add/remove button + is_favorite = await self.data.item.is_favorite(user_id, version, record.musicId) + playlog_with_title.append({ + # Values for the actual readable results "raw": record, "title": title, "difficultyNum": difficultyNum, "artist": artist, + "jacket": jacket, + # Values used solely for favorite updates + "idx": idx, + "musicId": record.musicId, + "isFav": is_favorite }) return Response(template.render( title=f"{self.core_config.server.name} | {self.nav_name}", game_list=self.environment.globals["game_list"], sesh=vars(usr_sesh), - user_id=usr_sesh.user_id, + user_id=user_id, playlog=playlog_with_title, - playlog_count=playlog_count + playlog_count=playlog_count, + cur_version_name=ChuniConstants.game_ver_to_string(version) + ), media_type="text/html; charset=utf-8") + else: + return RedirectResponse("/gate/", 303) + + async def render_GET_favorites(self, request: Request) -> bytes: + template = self.environment.get_template( + "titles/chuni/templates/chuni_favorites.jinja" + ) + usr_sesh = self.validate_session(request) + if not usr_sesh: + usr_sesh = UserSession() + + if usr_sesh.user_id > 0: + if usr_sesh.chunithm_version < 0: + return RedirectResponse("/game/chuni/", 303) + + user_id = usr_sesh.user_id + version = usr_sesh.chunithm_version + favorites = await self.data.item.get_all_favorites(user_id, version, 1) + favorites_count = len(favorites) + favorites_with_title = [] + favorites_by_genre = dict() + for idx,favorite in enumerate(favorites): + song = await self.data.static.get_song(favorite.favId) + if song: + # we likely got multiple results - one for each chart. Just use the first + artist=song.artist + title=song.title + genre=song.genre + (jacket, ext) = path.splitext(song.jacketPath) + jacket += ".png" + else: + artist="unknown" + title="musicid: " + str(favorite.favId) + genre="unknown" + jacket = "unknown.png" + + # add a new collection for the genre if this is our first time seeing it + if genre not in favorites_by_genre: + favorites_by_genre[genre] = [] + + # add the song to the appropriate genre collection + favorites_by_genre[genre].append({ + "idx": idx, + "title": title, + "artist": artist, + "jacket": jacket, + "favId": favorite.favId + }) + + # Sort favorites by title before rendering the page + for g in favorites_by_genre: + favorites_by_genre[g].sort(key=lambda x: x["title"].lower()) + + return Response(template.render( + title=f"{self.core_config.server.name} | {self.nav_name}", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + user_id=user_id, + favorites_by_genre=favorites_by_genre, + favorites_count=favorites_count, + cur_version_name=ChuniConstants.game_ver_to_string(version) ), media_type="text/html; charset=utf-8") else: return RedirectResponse("/gate/", 303) @@ -279,6 +361,32 @@ class ChuniFrontend(FE_Base): return RedirectResponse("/game/chuni/?s=1", 303) + async def update_favorite_music(self, request: Request, retPage: str): + usr_sesh = self.validate_session(request) + if not usr_sesh: + return RedirectResponse(retPage, 303) + + user_id = usr_sesh.user_id + version = usr_sesh.chunithm_version + form_data = await request.form() + music_id: str = form_data.get("musicId") + isAdd: int = int(form_data.get("isAdd")) + + if isAdd: + if await self.data.item.put_favorite_music(user_id, version, music_id) == None: + return RedirectResponse("/gate/?e=999", 303) + else: + if await self.data.item.delete_favorite_music(user_id, version, music_id) == None: + return RedirectResponse("/gate/?e=999", 303) + + return RedirectResponse(retPage, 303) + + async def update_favorite_music_playlog(self, request: Request): + return await self.update_favorite_music(request, "/game/chuni/playlog") + + async def update_favorite_music_favorites(self, request: Request): + return await self.update_favorite_music(request, "/game/chuni/favorites") + async def version_change(self, request: Request): usr_sesh = self.validate_session(request) if not usr_sesh: diff --git a/titles/chuni/img/jacket/unknown.png b/titles/chuni/img/jacket/unknown.png new file mode 100644 index 0000000000000000000000000000000000000000..92a72d65f31c3df93a8fed7b5756caf8c40154dd GIT binary patch literal 27489 zcmc$Fg;yL+({CUIcY+1?5FCQb5(vTFb#a0Oca|kVL$Kh^CILclcUasl39gH~F18B` zT%Py2@BQvM_Yb(|%$%w2sp+nk>0eb>b+nd-A_49T+$T?-5GX6X)p_y+jr*S)8}l(I zAUNLV$rHvW%5UH3`R5$w2IV}T&b;m@Dh=AT|H%Jk_^lkRjTZiJC^mXnxOVS1c1*8D zDnHRAVS8bFg5were)KP_NiMPdw&droZM}$!&Cnm%qaTIv@fle#!diA2|3oMd&{=)6 zZz&zWRa_n~K5o9~yclgW?TaeDpKiX(+mZ6&)y$P(lUV{qrjTMVCSo&vb`NKvU0eBV zql2#0#?nm^?;ak+6^}C|PrQb$DAwIhKG4>n+7j}}gRR)qxSm1$t--c$fWnf+eL}pc z2uVKNSb2Wp2quB8^fmgOC4h|nDd86Xm`#TJyV+1wP!{kz=vLw(Z=3S$$Q%?z^e`Aj z{!dX8tMsQ<5j(w@mW=4MtraL=2Km{#0X#Wa+3u;YwJE__EA7;4KWuCZ4`1x91 z1w?Kn0pZIBTM1)fF!)D$nXsVFsOi@2-iQfgP?n}Z)$I8fi&Y!Rm9rodU~rrp)%D>W zF|vyJu?^SMOR6(gm%iL7T}A1yJN}0ozE!Br)`75 z&Ft62_aY=c7bNeh`a<~Dec~9lGI4Em)a-D?0=H5S>n1rCYDy^UdG#8sxv|pU*mx$s z0tzyLLi-!FmVyxPWXk^O#9++pPd-M{N4k5i$ov0`rFHATg7&|1(U$+W^8b-yZc_R` zBO)2~e_Hzgm)P*IVl2eKU?h=y``Gq)_sZV0vJap1(8K?oMi$XN3~AV7)c?*FN~`54 zHhHgpY(HEYfqcWWJ^c7+qssM3YM0Mc0#+9wdq3wjk0};7f0% z&3*1#HrWam0-Z`_>v^_Yh8_4`0tm|s{OGoG*i5iw1Iq3BO#N&zYE5G(DAX+hSrtol zug}w_W!5Y1t=7x`w#*VLz1Opg3)EYBDub)~vgrx*f*58~?Vx&5rTKDasr9IbSYTbw zq${T$d#C9m_ezf5W~NYBN_Stg*S`NKvvaG_DuTZyv+}uR5cCT=68F!GRpwK2Uf>S2 z5qH};NQMVZ0K?aBF^d1ItLCd`Ob|_(TVf;ykSM;-yq>T^!Ojzwp zu{=T9`Q?%h+I?4gt0yrrs|qxOzWMeo_cR{ii`6lh7nI^QmuCasIBsxRjyGqvn9zIq zuCL}6>9lo;27ME4zEQfy3{UkQ+er&!338=${wLciD1N}^6~U)(y>aYM+~|7GdKv~9 z3rSzeLWKN27M>&KzH<@(2twitR@k=;8WIn3VVt>cJ12X9o|D!HyveCJFT@5#jLcRm za@Fxx*4_}VD_$*C2V+bi!)s+K`Q~dexVenJJu?uc{u%QuZ#O<}7`-~^PeV-4EPs4r zWhf8L*X7aK>7);|l0a75Bi9D3^h#Oc7&;cY=Cms2u#Wp;5_^^*Vo zOan#+;#ouj)t~}+_@xKPK32w7NM&R~wNwwUAPo9`;!UFY_9EHU;aZdX8-C*2=OENX zLxkiQI`N!<-7?ugj)i605E9~Hr&Zg=T!)OCy24^LGBKUM}M0GUh6i4@^ zfUJYe=OaaY5YHJb0M_?CI!a)Z5cj%^}Y+gMvL$Dgj?b1k^ z8Xkzs8RA8akc`-!Q$<3)|6COoA}vpZsM2#W z{>$;(0QB-Wr$4$^+-+lxUb%2b3MN{XXmhQ&J#i&{B3vwv(xMSr#L;GfH zxLk7*(e>*NIpmekz4RF0`eG~MivIqOAI?9^=QTC0-&L<~@KaKUTvhFAL5{S)^L$D@ z(qolc(BRw<&GQ(<6)&_srUP2kI-B$l62-go_5RPt!Rh?}WI?*du~i+1{(-)K;QRmJ z>HkaocpNba#hqQ++}1$e`+tQ{Y>H{xg|Wr{M@Qh@=;71H7^Bp|`!~H3zU+t7-g7qm zh;WZJx%Eod6b1bQ$Jl<)26B=m4*$W@N7W+QX2FLu`d|7>5V)}QmO#QRGA044spwc59MaXdXiQ6h$tp6w$kICo5jg99Ch%4?#{ei68V`nPNOAk#6%hZDy zz2G@ETD2O{Dd!VYOl+1_F1GPm#j=$yK86M|CMddf=1LDAR_m43z*DFWCXDCM1N`AM zVlc1u=|KHKp2Bky(@6JM6knT6YgDit2%-nStS)*ZArrm^Y1iVKOO7;#yS~< z2(^hd;3fhx4Zg6n?HW{hZL?=}Z=hz);e}Me@ujskR#rAx4f)(ex!Wpql{4rq{qS&f zRaV)P^xcPO`4fTP9A~E>2D7W>FI+jpjNGc~4MuJUR>lI7O%7JMh2;^$mqyrc0a2~z z=HlrI!vI%F(G)ZsRZ9oCy;f5g6Z9R9QFpN)3j>fA+U>s2vhrezNT&$~JbVYWI$Vg# zbw_TV>L2!e&A`d-{gbz2`dQjk;2Li>t&_JwnhTVOt}O?t*;zk{+A`~mmz3X*-$p4Z zdZbZ8v{2Tkeh&@ojE{g?x^D2a=H0~Ev!d#nw>t>*x~r#L7Pe5P+p2YqV82K{Qj_F( z>C5LZxuf|n$}T)oPCBwr}|$DjF`kYO$IUz{FqA6PGH z$-ku$P|#@l@R6MB2bkQ2+7eH?jRk6PcYzJdkgWOQ*K3I5vsJtP{Xk*$;O_8?w5e-Y zDgbu8h1cz7*=jwZFkKmHO19=>=9-5M6C+)C_aqKMe-om%a@e% zd2dJ?BNm*Ry>uXjl0{N^O&E6ZC(?1)+Z9$W;X6~^-rGxQWL*@?BePY@!|MKPRx(!i zfnqo`EKhqo2S5KfRR~RevTnW_6W4j}L(xTV?N2n#$PSUL)fx&kt$S@j95UUekS0skDR6N2Vo?G~{+uO?DOB5buQbDhMn(a=cA*+>EzX z`jd|W7`;BZhYp-@n4?22)MAai%yFY~XkE+;iy?IIZ~5yve;p`vquCt$aCJ^p_p@Wx&wRgB#Qu13Rn3Qm7rSblrqR*#T>JHoDW- z&+1ce`F@Jzt)-QlR^jfoB=orjv}RX&&%6w1;ycw3g4(Q^ss)|$QVcip90HH*p*NLK zD~4CcTp;P7_|zsZhn)`W8OLUx<+J_<@9jNbd~POmU^FEUmZ=z+d`FFfJLh}TG6AYv zYzgi+YB&&R(6`2=M?cUxJLn4lT7?f)h@G071nnRVjz7N z+tzVPxxL%Z(gT7O!Gr7bicF}58V43#v!JWX=^3RTlGTX>3(oahgpHeM6#IoTorcjz zmoD>+2glo7OK_B6Yp1BeTd0$!lhOn8P309bRMO{!?25!pnESefUBzvqwATR6^;0^C z&Yinx>&V{4w&j>i&}FYJQu{4*g{^3*dj}<;uws2v^>uu(N(-t5g1!_&#l=y1M* z#m3{+JGd3-s09zyti;Tj*CvD;n?XtW1+}cNI`fWw@dwutJO!N^Mj9AsI_snQi=FRc zzU_j{T^2ubR_W$Gs=d9YbjIb(=g+FRKUMMUX!~*?zi@-^<}N-$O*@Zw&PYDTAF|-d zh3;}f1^Csfbf4Q!L_V{nWiZm5tMzzZ_LlOnnc8I@36C$5PTuSC!y!|Ymu^mCjI}XV z?)DCh{BiE%G3vQuM0Cr&uQy~|{AMuALLW%C*P4G*Qz)wvO+4Pr>iYiD z#w23v`av@O84&ulLWg-NdG6CN9dUO=LHg0w4*P*R+zG_Cbd&67jGATEYt^*-Z&>l%a$`s)B$1l2MCjOcuG}}Q3nk#l=+wzQT6L(G`q(fUF z55LGNDNjGdJk-*tL6L6b>g;a;GG1@WdM=!JS1R?hKUgeTpUZ{y-Cx5tmI3%<0tqXu zAp#ytNg_(r;kT{76`K_KF`tM&>@xDXyl%D}Wt1^A?mlrgMVi^=#P5%Ey7Q|3g88H< zLTUplps6G5si$VaT+`Xv%~$7PchY@9E%*9R%`%zc<%A=2R!SiMXExE?;k-JV+6z)B z5>fdJF#IJjnf#JHU7b2KC5Z++HEgPReSiV%A=mty9BKm%$l7ISw&X6U2aZP#;qu%qUlZj+D=6Mlj~utfWr`B6m6Vtgck+4= zg!F{eWkbYIDZVB($E53;x5mJyh{4w}L79O=-n}j~1Ul<$L#_?Hrvo%g+5}{K(xTBGikEvz2zX}_}9Zb`J@JjQW3m-H{x01T=J&Ce_Ii+I1BMYl+W>7+acX>`4KM`1{EuM?Q&{=ljLk1f`Q|0_@Y~9sc zLdr;$?TPCMr+T`avr}4?o#-=j5aN3YsU4#eu>GCW4@X4hg-L25?|FPGjM+Ycaub2f z#`TsyWdZ@FiV{WE<^4H}Cf zGoOA-{7zKd9;$3Tc<4utG+EsX8qB7W(R^0yv{ z?U@cTo4&}Wpo`jRy;;55(~1Ze|Khy;EGL8M_OONTC0AB#&!Mg<`wQly?K{*Z&VJg$ zcC18G0KCqc=9pACe9k0UP;x4KRp2-)Y@%Lyh`+U)3C#HFxlq42q(z$5u zZdX3!HcP+SeA9ER`&{YlMSMc%9;ZnSCohC{ILqEo^V)o7vR zWV7MR`k`9!I|$lkyZx-#Ciyz!P<*x4sdOmTW5~6hlAMbm&A_D5I{JLAf99Svk=SAq zU&&ym;neI%+h8W^iC!Gv)+bX9Obj88OO9Ng^Yh{rgiM(At+BR)$8wT^tmkDNMMZNN zAYU;hT#S|_gtK~`;zFoOrENG7AsT}1@Q7FM$N4(Go~j*bZ{G z_VVVrrqY2~4T!s*;4u^wnmVDbqSXN>WaYo6cle^6$31!biX74L zH_pd=*$^4enat**2y75yW3&p+A2KImC@52N%#!rbFy<`sG|##JhHslxM8RjhKjp{@ zbVOqTUKHXlfZesfFuioUA!%GSGF|&z%lD<8tEv?$ ze0)L;#r8|-!^>4kHVJIEKd0pRd>ak(utL|xckCgKoO`{_<4uO0gQJ%H7q^{fK~Mbn z{P8D2nyNqxkqFrAYp>}#9r}lR<=4Ego83-uMZgG5d}7AG@dEbQ#AmO$L<^0Zi>1L# zE9yxw-44K$a|X&uUJG0 zd=k0F0@#F|1jZg6G}O*7?9ZmBd{s-;GO*XOAfz;Jq$R}#vf6P#qEo6+M0Qg5N?f&9 z2pyj{EnxLNs9RWIV_0o+*=g7&e&Egoz8eciBNGRIClbO9a-1{3J(Co9`H;i7_<$Ok{jV!C-(XhH;YW4QRXoA(K( zDoVFko}Djz3vofow)k={c7Dm->HCgw)dZcesOJ8))v}U7>*yKey0lE@t2GG!??f*PqB6e_RLFThw_<{<;$|b zTKWlS;qx~?1skDl9bU&RIVa!;&*Qv~FE=eb7?n6X(Usm!I*uV8C+1kHz`o`qZ}$$S zHF(KL^N;;nGi@JnJg{V@Prb4oZe=>KgR6M1cGItZ4AfB5;Rd5?0H6cf_0c^Rt8(A& z=V)tD`R>f?4qVxsZLN6ij?% z_UtbRr`#=GH-EU7%PXyLI5YRkqN=V#e3l5vmAN4<2*ngkoM42SRmH> zTzs7x&@xGfiQe5W|NTCQ{XK!^-d@7Fcy8zo4`%Xs<5$vQ4V~_G`L2<4b=+P*Ee2*TiO!`LOfDof8G>M;a2a6cLzVUkbU;gbkqH zTEP;%RB>huvcF$={i+P0`^d1|9BS&?9(^;kG1p|p1-qL);FR1GPmUK#d#k|T^{=Ef&fez9T|?wEe=GFDsm%K{C(1OlVn z4f}XZRPl0JnNsE_3`D4e+v4C%OYs5wH|AE0tHiB0H8pGLOJidy%*pRyU;Boz`eV z?H3rj=$hNP&PMQy}<`w z7&th@$+x~XYGmp!;#g$renTn;SN7{Aj&Fm?U>?g;T4C}01dxj5iScGg{2Au6QM%75 zaUv8-v){nKiV;?<8_H>It{%DYqA0N} zC(ZJ2#-17~JLZnlP}Ml!ts`5+&F1_{=chq~ppQyFpWB^;Q3pM5&Z}n7!j_rnwM1Sy zI1R{s6Vc7LBoltbbG--t_7$}El-uJ4*Vo<;ET7iel^JC%S=!mPn^@v}-TwybIX7S* zmPsn;z0&@+aUIJE)%@i+xkyXxftE^XHn=&$D>p27#i17GuR?7s&O$ut#uT1Ht5zV3 zqiOo)rZ5@W#WMkr3qBHgF{$w^PJN|(@36y$m!Yw<{RD_ZTG8Z9I(>P07=kh%=WdPj zh`J~FIv2^S`EK<(Nk_J1YqCJ&M#|>^nD4s`X$n4@oT@Y#t6_7_5WABYO>>}L;@fL< zS2Q8AqSr{)@b~0|KM5j&Dm!(uCOvhQRdF$`oRu%;knOJ8JZgjKMKg)K!hFifwK1Ij zvsWfbh*<}=EHC$Wp(rXwuv~#Ry0FK$I5%0(eKo9$RpA9|L2WIXzso%u>*=2IwXRE#zG6e`ja9)%rb~6J-m*1-AiRO~FapR3QPJ~Htf~)Q z^tv3*;wu9d$PZ!NJJBBQK?G_UFca`w$rK(^lTl3B5 zfM6<2B6cwoUn|=yG?Ae?OHK4Nz_s8WKzg)WGbB6BxD>gZpN%Tog~-%~Ymbo| zJA)wZ$4f76N3NNA!aW#tE+Z85;;ARvA5gllisCrV)4@ut=apH#OF_pQk?K24Z1}u< z4rb^4_fi*5O5Dsat3|d99%3PU6&;QCpQJor4Am_CenVe!$P)WK-+#esQHhW6TD_`< z)he|EF5+p+E?$t6Me6!)M~tF$QWtBVAp^WD+%$kbc?aBlL+_F0OBsLHOJ%+(ZY<6r zGT+^MZE4OMU90?x^2+C0ULE%dYWBQhWld@Zv?T1uT}p}VxSg@3H^bi}n{SL!cUx-6 z3nf{2?fvh2SmIaB0}US;)&BX@R%dI6&GVK1BrJs(i)S4A8V7vwX*?_zuDfQO2R zH;Dn8BO{`aPn(I_>R%^T?rsG+pcav$f~6~yvX^|K9$Tf9r$^1OD@9byNsi)C8UT5j ztsqD4XcnZ`wM~WA=|H_bj&zdy`B@V z)6@Vsp9U1BLBp>4#(oEc(eLpJfez0T_1JF-o?mj2qSM!-7mS6N3TzQyD25g(Pg%NpO1a=(`eq)hC&QIl}QnPZmvv zr$}f}RBlBdkGp&;z$u)k5_zoI+9{oDl4A52?00$E-2Xb_n2D!*e8soi*vtBpUE*62i27-xX1FrCOZF3A4plBD@jt%rwU2XDm|IvIB1P&7=W@WyO&k*M_xxo3$-YYFSN zr}YP{4B}m~If9?Nil{Wgn^vaMvw3m=TAue)6b2}Cdc+mhXmC>pAWAeFfUBcpKKs@# z7{@^`r5btHmE$o!uf*!+!qA<8!{UE&PyRY<3zSiVuAnT2Sc@vKCFs?~X2tx;o*8J{*?W8_`X3H`paYj0H~xf#VIR+nDW2H;u7rYtmhGszy8bkTCub zlNTp5GVZ)UGhj&n9H%cEp+$Q}P>ibjW^6xLXHu}iej8C7M^#7iCEPQSmEVvgFkSO> zHs^vzhJGq3)FoeJYGm%1USFI=FzVzRn2aA5K!-Wr1#)=y+AzX368B-2n(aW@p0AUp z5Q9Z}E#VZC$o+DI-d-dkQ+UY>FEiY=Qk}D>5fbt6_2*XoI~CoDbqPjF-}YIPi#KF!XA@zyha5?zQ0BQAM&ssI?y0Z8t~iP9^K~*?3dR(<)&1mJ zgsRK@lo?~k>-m`RdNIy2>gwd;`=YjX)~Mq-Afe8)vtEl!;%49>%F~wjK7=910ZB)u zYbO=yIg%7qMoAq-BxqC~c|AMugVnVtGVYsqLg#4&r$heUl-w<;^)wD=tJlZdv0m>Y z1ur6S98RKkk`=#I)}CFEAXwW{zSA3z0W>vf%6f3$86hQv3nwQf-Xw&WYL=kC^xC`} zhLHmX5BX$zSTrZBNBaG!{JM%iq@46ZM55ffBf5cm%KC)I1YZ+?u8MR9T8wq~ z-s+cVH~=9x*21%~fR-GN_7bC1H;v$85E~glIy6q$jo)Jdd#1ykk=3&F!FfLxBW?um z(kfu#B96!HZu#!=wKGlfX#0F7sX?sQ`4_H}RmSK`zdUAYc@?FcX^4Phva?&L;R{w9 zc3E9ju8@W7Q)I*}4#h|mxig8@k9O5v+J?H^f#zLFriKUtd_RqSeoz&AosLLp!snlA zxo?+MY3kD!HbicP92+{go3^u)ye9A_98=}PKQ1yu#5HP^M1-yrbxQghYOd_;U>r;t zAp`B5_D?Jjm!40q7_x>R|DBXJIAyC#^2UL{SU{H4Ud^kH`-xYQ#40;T)*P#%Cu#Q% zgNheU(v#!48C>z6RtLwg0U=gIIw~r1GCw_?d6kBSirT*n4gkBJs4~n>LQb;tM0>SH z{NRf%XXfLO*RfIHy69v;agnIPU;U_W`=JI-PKNt|_(SKbJ%Kk&DyhT7K>df4`+-6w z){c+|0^-o&VJ1CS#7EV>=av4;`D^PJQDaU%e?W}tN8^F76xDyg52Fu<4yV`CwO)Ka zCN;V_eG8EL=_s226D&8GKHLc1Ca8zMPy5oQ#JF*8-0`X>8q061no?AV$>`>5MWwd< zbq%tw-Hd-@qa`o(#^Qp=v4lpK`{w42nvPyhmU6ApMT2WJ{*U+`lZke+91)9oW0h8~WUl%MRf?0EW?}aHR5PDpJ>CcUhkM9It z@#XZcM_fOXu3Vhd$N!*!9&)YjXG=~mfj!hP%}(PdnUFhyD6s7>RFSf?n(TIaEp(Eo z12~$(O*? z6X>LAz1slBXb~d$ZdDgMZZ+is@Zj-ugIyxORF$LAX^Wxe(vVB;9my~eZ~L!lE|LP< zgDhFVCKpT^Gk#^P^Up)yMXx?L^x9{Cf2GJ1j?+sNKZEecJQ3x8I-DUf*8KgrWdn2H zcjl@4ZTdE2p{^7$Lh6Uu=-jC!DFFTGQ}l8mK_TzSndRo@3F8KS44 zv#cysW5-~_%mF7AA~6JUcZvu>nOsB(whDLeW*eDNJ_fVF+t@nYpLNJE39p3aHjZaJ z=65g#S?yK-8hZDg8r)yDKF#n@o=Jd60Z;o0m63`r6?yi5^&A8@;Am2)Nj{I`6wR~C zm^Z`bG96jLx=PGTAI?+gd+E@As`N7#7OeLwqS@tUX0Locy4)(|YsOhIJjV%?UAtk` zkQw?rWA$MnrE4^+ZmVY@Xct>VpLHZBuhjZ$AvKE0hC>PAJV{x+b$hM_OIs^OVIVM(F>WTv=Z+u zj`dFo^D5<6R*A9MtQ(x%-)}!wmPy5?B*};A#ENOknL*B8 zyGBHTG5OqljyE`U*Va`nZ`h9Sgv36F!1v0{M8<4C$cMrO$vD|=A4)37o`2pn8b@rZ z!0Ap7)iDTua6wwKWEnq4EkN9Ft?^8Is!#95BG$h-40aHXDCeWI@q9{9WX_c6XY~jy zPK(Ciy)8R$^%NYvUwBi|zRgI^;8wVIgdsYi9FChBD~45pFmd+?I${v_8-AHuufg#e z|M%WLPjD+7nQKu2DuLD|*=c+-$!z}oF&`hS$`efQ8u0YeqT=WxNw~h$$o!x!?c&Px zsW9&=(PDwLW_bAXjC{?^AfD06JAB%|>TV7h!Rpk>6M)=(;fZJaL|nD`DEKyv5ICQQ zqTA>!`uX)kLUIy#v&z5%wvX^u%qb%Fn&5(nX59JAwXi8c1Id>puw2{OYzI24NN|4U zo*O0YD^fjUnkXuTDcRz=Bi6D1KnA|Xtyf-%_NFg%u;bTx?<99yul`0gS%~+a5GvNqc zo^Um_y4Kdt@D71%TQLTcINYh()?h?<>mO>TM%i|foGde<6 zg|4${qHdL-Hwmr3%!NiXUql8(Ze9n|TaAl9s^H;4o)6XNu=P9vhY;6mu8pcg%wKuaS$iPpXORM1@cqSycH7$6+VA|`el9}zV&n&GG znrm%@s-bX}_uQ%~QC3IFHi2WM&kJ|>9p8(LFmC5(Gt9&BB2p1jCJHsXmh1KENR zCnV7!7@XqOo5G_+SIkr?ieQD2@3$gU#yG&rR}Y;pXb%pm=myi81%P2L&sfSSN^XUb zYsKF(EHP$~S8R;=``T-gfnCwD5+n2UZ&m!u`}A@UmJTwfb}zvD8Y{Ve)S50Xqtk{# zTt^X}OtySGIWC@mPj!?eXpFLsw{h&A<_6NHgw|#Ly{O(+_?0!I=A+IzGK+JZ@e5mo zyYH$zDJ@JSS;nyjk)sWC!g?eO&YdY1jH@5Xqju>-_>+tTG5~Hw~zn?ZUT~sq) z99f6gK^1Frhc_}y!TB?gF{{IjgWLIY2gOQc2e%uDGsELJqHN(z=a+2LjjC4we6MxB z?kvGfqED59Uw>=%dIJ|Pd%yxdCvODE9cE zGSWy$mKCCyx|;6oPF#{a=&pN{=14( zM@8UqfvH>3UfW?&7#pjOB`q1yl`s%}SaNyR6^G7$yng;hism`jQj0EGAO{-YejyjM z^CzGT{6U^EYEf6jWnm~hI)CnR1>;@b)Yl&E=>Y!_zIv0l2+z+Qc|g}2+8mBBeuymi zj|W&B8HK*ED(SR|{qE>R*XK!kyuwl~aEN#m&~Sy#<$&DqXaRgPJi@xgjXt`}K$$s-`ZVzAL)arZP-(Bg4eIA^Lz?-LR ziQ(d+lE1+Dw(D=3Zid4^m*Ef5HIoKQE%#V zq&qtIC1?TO^>QsB*r+b|zUqBp;0FGy$fFrsc)l`6_S40FkG~n^v~wc7MJN#nX^Z2X zg9sBtn)j7EATq?EfI*0vdfy^;(nwhS{P3_foE5{_LZG@_uo0ukPNY8dSXCh8N_hALXZyGM&nGxNnod@W z8dRi0)Tl(pyFo7<)F!y;$l#4lzHPGp_O)y=tJJwxmR<>j0V1tId3k(*>4^WLoUC~r z6=Hc;E6iwhq#dlnu}Kq~#gFb6OY!0xP7<@xH5*&OTa}(}xt?*bMVAsQRW&woXfx5? z&3wn6atUu#E!p@rrJoVQd!+OOB44WJ-s-sh9Gi5AWPB1aP|K#YYD$*3I^rPOC_SiqVT-4HbRH5=;=p!b=%90;CM!hJU0#OryIA0js62zroY48-)VG;KhZZbbn zcd;=1p4IU@00^B2az3pFZm_)l%fmi&b5sfU{H!U~!B5ZZD`dUEQTM%KxG_A~w#I!U zSCcuhO;~XPrl!`N%@Qwut-V@;x`7{VY4`yI8#|MUK0LF$P$;s=;^8NTtt5FEK{@-# zAyrf!H3UjY$MN4|j>p|M2ymPly8HCYp`S8&*raZZs97y0(fFsjBd!26KE`T z-d)ZVbWcPlTa%04PpWm83--N>tCToA9BvJ_0--=gO1$*hn^=bQ_}hoFsxy@g4G;4H z=LWe?v3S~T_uyUJGPC$_zl1%?Fr#%duy0{4zt4@g1KRqa%E|IezJ}zAmYGvQEX(^P z1#TmMEXWJt6z%-T_4Ze1{n;$2*A^hnhF}R?B|1wRjPJA+G*N)i;%7 zTBjrFIyHs%TpK^*Bh;AOC**z9QgSVuKm3(AUNhUi4*S3|ng z<;kvMMn`E+Ccr2Cpbb5CQ_##Z6CK;13y@XY+LQfWP+o|+OGo$GI@V5@F2iHo2nen@36iWB*o{{%;K;JQgx)q@qpk|x#`4_-wO5Ms(tN+ zpPkwZ#~Qy@%M)K7VQix#Xm_=(M3=80aS{{2f{r0SMm+ zhl+M$J#aQPUiXbT)anVN|3<=Tf4te&c-!bYR*Yp!&8OZAY6{ikk*ZCong*TGiKNU_ zR=ILXV}a-Jc|IB&>^Er{DydGWwtSyr1^gnwyV_N;PH>vdsNNwA}I;~1ne zwk%ZavNb6sbH$e-X>A9ykJos;SNN{dTlu)XNT0+MK3-w4GWR0CMFd4iRt5yBPv6w* zRnp^i_{Qi*3hTrjKBVaBk6EsDGdx3QTX;~Ls}R@JX+6^vhZoCm7xp}{bY8ZA`W&Rl z8mb`k2pO^-h@Sx4j`80G|1}Gdu03SH2TNY+YL?;FS#In)sloKOE*on=GKLJfaboH$mzAb}M5?utX_D zH$s#zc@4Rw_Kye%Pvq*+pSbW=3YV?pBC@43w%NvS zh^4<#vEMqpwr27$nfW6{g?5IW(qz2DgXmSX>ijm#f-eBfj~D zF20u!)JCVWp_kb6*Ae4jyL_C;Zo${uEi3uuk>L9@mp(NL25vu<;Zu>tVErMP8eYMZ>3IJX6Fc@G~SC+~wzb zMHPSl70@D(?ADgLIv@r(s7@!8W-Acm1Kv8yo4@Gk7mVyR$={AFklI9X~G%#HEnZ z>JG)qu_HMUrQxnrbsB9eK3zzisIolMam{QJ*!4;6a6*cc-q0=^1&AHoXuaNus=ws@ z?v*F7ZwzR=>2tWq<77?fXV*l859J4}6J1va^!VDVgTiW~o|Tr?t9UAEm#|MIzqtHz zCK=#+{*q$3`7NIwMxc@VF2v0?HA4O(zD-y@7^@Fsm^8Q6VPE? zty>&!N^l<)#gx+IUVh}FCuv*3c5@;7@N%ka8}fJr#QwmEJwL4GXmrH7c%UL-lO2eNp;Ic$cCS{RiUTj^`hgg;a`%K zf7jntsi}UgX39q#b)(q$N5SYHtv0>~4v386^dP2*9^xS(CjwtX&C`xqjGGm!YKFR$ zsq@Q5kvd0RENmI&kq@pJiaRcTxp7hH%ewKK$|5IoLkKU$PmWGG@wQC4K3Dl1T%drQ zi%{E+_O@Vev7sJzMr9740G;7t|Ip4au=@PA2G^VEW(&%mxvmka4s@nZhG0dKm{3 z9tpTr;)l>fFtPhK=dJ%qy92dPmvXH5kTF9Yyz;gl^sqHHz(|n>L>xRGW61v!e$jdm za30H}M<7L0P%x_ENY{NIt(i31rU-xf3(*U{?CC}lsHKE`yjXj!rD1wdHOg)4!V;zF z3YL(mE+Zi&P@^?Q>zwOJ)=GG@b+8sjW&i4F;Ul)oZ}F$b5tXGq^gwH6#tCjKal4bp z!Fuw;*Hxm+y?Z}DL870$44cCm7tdoy(GAopWX@FIuRDZ3dg$fxsRoiemWH2XA*N2l zD_7}Ga^E@vE>AE7p^2 z6F!X;vvXNYwcL#s;E*OYeFdaVOS&L&usS^bKBj~-a(~CjSo|2&Ch;5T|KC^ueag!y z=fiJeIO^FsMK4C^eaaqPYB}DeFErn&Pzf@YZ$Z?WHSNMz$J#N;Qs^g&VX`4B6Xe;7 z2uzC^ApC&g#P-kRWX5gZK5YO|zlkudMbtE+q$1>_raF(EcKx|N-K1ATV5P*-veIRV zz?0WU^eXdT(--SY(eARnJKk#qV*}K-*CziUmw9jsQq&LzLQf*!!;iF89G2G_3<>(7 zPPIq+Vf}P$CShNzhl9-z5gYFIK%cLZyi!zcv>!6SsO5V3`I!vM-QO9&`|r}Njz;a- zIFl1v>ITIC#fHbo6=_wsJk?9Qk$IEGh;o{S&=j@Y*BF;yRWN;=XLPSaYi;&-aYA~8 zMT6ruZFrQxsk)H;))=>T;UzxUsyP4^CmQ>kk$Xa-41JNNP#XVLjn-is&8C)&`{1eD zXN_Evwl-fZMx%x;nQe1m&qAbAdB!RIhE*(J)%`SFnbLjogXkZv#^~kjQt`_a%Y$`( zfB^QC2#{R&L)}%@_)J|fg>*YL-}Uzr>J!7q6#)CH!|i`Hb)Hd8bx*ig0Ra&ZL6EK@ z(wp=euz@0=AYB5e5D*a|AUz;ek)}v5QEAeNUz-nGt$ zoSZYWcapO*zjRcDdI-}hyx;6AiN>;>Ke;c^w5%e+iZ#W%rr*wVU0 z-f>`yrlida;;?5@%}$JC?M_VeV5pPGL^jG?K4o*oZP+3PmD?Dwy!bHWV8s!g&xX8n zua9QD4Wh2@4rWzunq3tyNfW4`S{`qKigHq^b1P*xg%_-~3J229%Y;$~TNf`xheO#4 z$C~3c%Tn=|f9d=hjPYU%MS6W$PrAjBLuBh*}0Q$VabQA4TuDfOcTftu$ml>qYWKr*6 z8CxujZ>5u~_m*6K<5W(@p#cxtcK0>I4Al*yu zRAR{M);X+1EyeNeC&jhpl1D(5REMf9P;kfVJ7yP$_DyCoe`b147F!?IzsH^;r6POp z^25b834~2{+7b3eFj#26<2viUFAO~CAjMDuKim`p?Ckcwdte6Z8CtV9Z)z@#L1mf#JMf~1t#2gJDp{nEk%OW9%3h|QKy=}lzMxE$f)>P4D z^-C|4olNniSHd$&!s$_Xppk(v?%b*`N}qq zw4()c*W)0yfi@dlERMZ`tu~sHtK%hX<)*J}qxWUbR}*vJCj2|knv@fOrMdo z{3+XJi4#(i2lMO+H~ST}5DLEX=|ruz{GdjD&dnt!@O6pfXODYI?uQzzrIg3qlDg^< zo|!ZCHO%+s+cqtYV6hbW=Psic01Qqf(U{$0)M=mg9<(`GsgJ){(=n<4*=syG`+~}5 za7s+i{IlmS42~oasAGUw1&LnEq{S;$Srv7=A0K%L>}r&7Dw~R}3HP7-R$sqDU%3M= zYPEj%%ah>Tt`0N6o1|RGsbK#$v#nB;l$j}Lz40iXCQ&S5n(Bx?==G`!X7gT0!jEQR z&$V&qyg~;K^23I3F3%+T2W)lG#H$d|OAYl)d*NCu_|sXx?)jw0L*@5w2D!v7=ct~1 zcIgY~mZn~Lb>|r&r9FpaMepT{YA>HZ!as_kAKqUKK5gzqiqrL- zS>n-J8g;#1^d;%=>ATtH2kqs^){b)IsmEJoPyNRj5#QVveHz-wPhaRO;1_+a_uy99 z6)1kyetnJUbRoB<9>w`2gl~pP_NS+2lA)aD*Rz19EFA#vD?^64o(g}Tu7$LY-`$Xk z4$`4bB)4HALGU6=e4+A-TN*{zih8-#%|vb_EhgAJ`UMzV*b}v&g zpMStN1Okb4`E=z?0&;^=jS&mDb9)B{w>^L8nQ1JOng`d4CQs0%*Q)$<`1$fZHo2u?D1!*Fr({n< zI2A+uIUk4~In(y@mDz+9WA!kSl1n3om7bRof4S+zVzd4Be7pVRSaqnLTm zTRL~^XdGhz-FW3mfz80X{sLLsMO5vhV9pgJO^~QYhc&I7`tit7g5#(kJTg$jSZeK+@7h4PuClwPuh=7^yZQN= zhmBsc{OjRa?X>F^&z~qprTxmATjDY_O3)1#X1|}OB)9YM*^8E^V&iYneYoY4;@Y4& zZP_2-Q^&hIJ*t5w9V8XXA-Ko4qDSFSmKGQ4JQVwEnLQ|~k+8#fk`H;2eN&@#tqdQ1 z;joP(m?+nTydF`#0r)T_@E{|`0^CzzET7J(rFt0NYducstW}9I& z+~LcIxTK=DA1+Yz*zUr8DZO~gu60{|9G}`xHa`P~+_-c%lCb9Nxpjt&R|reH$d}!7 zx;t-7!d|RS$&qjLuY^JPh%wdf!?@yrYDMnP=pQSC5+aM18&5(G&>YLb8_PP{#o0C9 zIYi-Q=!caLuJ=cb~$JGbcfyCDy5wQif*W~N-8X?xZBdhnX} ziFoufp285xhxN6P%r_!`PQ23;dKFcA!39w>n9~)Gf?mRH;&V z@K+wPYiQxEXtr+8Lft!Iscl+p)Zn{h6QW$OUXVa%xkkWE*}FU1>*7fY-_4y0Bh*Iu z<$1%UNmFwNE37hat^+R^UvI%@04sW^_l3X=4emSa@P zyfl5D{F|Th^pkTov}4C$%qY4sq43Yai30Nv{0F|%nXP`Wn~TC;A$1zEZ3mwp4af#5 zm6g1Nc|5L^Pv07RFq>L%V(=8)_)9X6K{B2}C(|JGs(7mwnemk2)pKclA#7zjM_7>V zpuOILbTd8^S>!UYUrZ8fggS0lLErPHDQrEnBJ2{}Q6FbeZCm}VVe{W--@u6Hodf20 zLpbqG)o^3Vi;KAJqdRdF6^AhR+J>M~MkE@&j?wcSL%w_M=Wxxa40$LdZXLK{tr9Eg z0OfhV5t`#cjX8Mx(yB9br&1Qr2qY)qv>peXstOOiyng%QJpRLL6`yNK1oc3g?t2TH zPbn!HDk&hBp*|_FwuTXJSF&9*S*);0P3|p#SG+e7es%-Xi2^1--#B1iPD{wAmf^Zp z`;k&MBAjk}AJoTGd`01Ng;qn&h5jpkZvy{zeD_R56BGB_11X4`p5xjU7a=b@tPiOZ z33b>b6hW-CiqE{Z1)EC;0xK%+AZHA*A_S!^>!_s);#dH&;}#cBC+T4-J@FA)K!m3aiD{{gVL8S_HGqtiByp$qOitS{o+}QEzm@ z{+eZ`P0vIdsUV#~sAt=5Yv0oCGw%-St`>cWWcXmX9C~unrDa!l9kapuo}qzu;B^VG zVZZ1Whds&|g`GA6sKm6s+RAF$R#xyr!D9CGI1wAK3fi8n+t*!C2yasU7HrXCDhUkl zv;0189u&`q$t32ow$F5qbbo$HJB|$yT}Iu`iQTDdoyp^K!pjuc&d&?=iWW*7^dsxa zH5Hs6k|(=L_ioPrK6(XhnyvJbOL3e@wKM|mbdMio??wP*?(2=qiIjY2ULKMqPhy1B zmY9YQV)}YyCB4Aet*RAgJ`XgjZ!#!}Amiwb; zb8&KSW4;!(9SrFPFEXc29<<(e5x1pKF9!%G`lfwYOo>A4Tf&F3)gxEK)mA6iI`&-O z%)U(`40MJqv6=F-dRdog-wcB3(?@XvdC z9L48GcnwQRh7CH$fAn|YSegI6_dC!Sfa6w(pFv6oK-!1aBmC{XVKIm!PO^k1YS2s#OG!*t5e>xG0 zbUtq^ThobuQTWjLhp8X0-}pmAUHLf#{RFCn{g$5?bZZ(@EkKvL=@vvAKkZoYbN}wK z5(eA+eQVxL90BNN)p)B#zgwiNvZq5j?LX(lmMCxjbowFY$PuRC{Le#--CfJbyI`(= zKD7R?kM)d>xPiv_ffn0q)8wg#{eNn2UH#X14tYNevQ@hK@>P9z8|!CP&Y|(ff}L&j6W! zGu+!mB+3j8u}*x#Ep*VW|hagc}_VfSc}EjF?j{NB8>^xT1`7Ckcq*+-jF| zlI;Eiu>%3x$-SWvn|Ve&jCC?%W!lkE8Ghzvvc#CA?6)(!eF9pEtP}Mhl zL7L3NU@7o<^aEIc$O=<0z^taK|3Dy-?`P|10@dvH?^*CWzLQ{~G?;Cv21+AgRJ@e5 z^+6g4wh0X?N?DT z&_wfPHgs=E@({@f0GU_4EUb2Q`Zq#kWK6l>`ko3Yt?H@{ir_Y!dVgWn zRu#m3RjTk(Zcf~>_l?rl7cetjK;8W?8N^fbRTlDU7uo6_Hn3SIX?L1|jlX&abah0R zAWy9Vp>K8gG6QQ^oXjNFgOg|X`^!v#fw;x-kD}N+h75&sn$d%)DP$2f zlF*V?=o1{-nrV>0H>H#&Wn4{r&bkvEp}E=Ts`;52%o_{wlmwpe{|rNWV()icd6)|$ z%u^E_C~+=`0$ib#+?I10~RK)rEgLS^vQItGI>0myvYsBNVbbY~&ZJrt_LT0TKptv4wu*Tk&Fd<4ogRI+ro`aYwCJ;WcEXT1;pQHM5 z;mfDvE`7TLg*IrO;Tl`NNlE|p02B474K8~x*(7i#i>a$|{QT8cy;~6%-$Y-!s}0fX zsf|e|UzJ8%cnd!~hMxd4N`ZedS+|o9rx8cY?={!kk(E^e!Ps{LeG|%uKjMQ~!erqR zN=#n0h0wyCI-&^ME{&VfyOmba*ta2cs&ena+B^m|L~#2N3Z8@f1}KE*O@Oct^dJ$K z3;yOoxC#t)iNK#)*B*;QslhG#Hc%Vc3m2oCRLu74q+d?4UckNHgkgl+k!x{Lv-LhU z)0t16)|rvwgRW(a?fa=!ezDphH|#YSIA%YwKLONt+MjcvStHZ#Yyn_&WPi1U0U+`#vIoa2 z4l5Qc)_5-o*~C;pvqq`>hF0!T#uo*cU>inrfJTw>sn@ImzuNI%M3J(rL+3|@s(P=U zA(<=oJbwyQNT>#R)^EeIf8 z^AT6AikzKIf?k;97NC4Br|YH`O-aE{vt#GfeMn!L5AVjI1i*D?Lr1bo=HiKC46^39)q0IGL(?Afj}Lx+&&3!&W}qRu%oKLn?T9ml8ES>Ma%@6Vwz3^n zP2YVIyQejHw^k34&g)mBzwo;n?y>A?%0q&AEQh!Z^YMl@Tx9av10N6U2iR1MLMvNM z5nmM8#|D`83mPTLdfl;t(3E5|;<=Ivp>}=vm58S6_m7dY{tbx|6OiZI)udB?KR#0f zCzI|v005F<-cGT|llMVvtD7gl4^BGQ%i-g?kYj(s;R%Kj$6xO&gysynd$;A`HF-j} z0=wd4)yY4?j&7C49z9%Wnv&>Jt!&O!XxRNqJ^z_Zs|~NC;Rv1x1Q1sdFR~3q<>ISxNXXM|37}YQu)G@b@;eN z{~x0?KMxbO%D?{pf9$gFaN~=^*T3e5-v*>WG-&O$|6TcF+L<|N7JKTSP5yXSOB6Bd ze@$W$Em30F@;mh2J@ofo(JIH9{cA|lPpPPxC%3D6uwK{Q^K+#p(|LV?EaEghKfJOi z(1S|aVMoj)z+UudtAwp4K!C;Ejebpbjyt|D*1G!=ojFqsu zZGprqo6j$#KnJEibuVMORUE}#kDJP?w?4%1jpp^p5sib3yyeA7HprE0Zw}My>l<{( zR*xHXl4h14iJcy=!49tQX29}q3lP={>l?Q+yIagQ0$D-d$ssXu{%c7VCPyX`2&oK< zEzCf(;(F2=JW6aaw6La1_5D*C1y2}-N4*IYY^tm-)yvyiL`j|EB6k9+FqQ{ zBx7t@shvTdrScFLBgN%ZPeivmQ8!iw-Bs3LBDZ|$xR`aU?ah%pu#J_mjJ~?V>V(vz zw(LvA0uH;Zqibd_3MVqJsox+!#`15MeJLsOoZ#=YNk2)4Ig!J-<&Hk#Ge;_DF6 zOMZwCZDvX?A};J+7J|ED;n`0u|0a+--g#}$x9e=-G)z95den*^cdX`~T#+X|IiZ%R zu;A**q)-fvIo|)svx&+#b> zRkRJC!W2N^(8;sExf4eXR+EsvSOw+*QP&PV+u2%OE`H?u*|H;7?hxSZl$d3~v)3~- zUPQ$Mejz6)oNE}l0MS*>-i;ijb%a=^sf41}s?DSMph{ohMu}Zp&ZI;pz9|_S#Fpl( zj(K8X&#t`=mZjluJao2%=L&V54-3}5jX3o|5zZEdt}AA}D&u37)S3M$g74~us~dn; zmjN&&NWWIR%G7OBVcnerM)a^J4jvVr{lvF__zuGRUU&2eWy|=7M6qspSHI|@OaH<7 zc!R$w#q#PZ(+Jn*ksr@ojd)@Qy_*Zp)!QUI7voTNpPcdC4|@y~d_%*(MoKp0({SoM zRmVC5J!XqrZ*b1@Lx&=3W^^_Xn=f1DrV0HN?E;mCPiTHIb)*Cm(VB4+n&HFn+zETa zy$3*dJ;)g^=Q|F!1LDdbv5);01E|1h2&^TCZJm+8<^EV9mhONeX{99%3P z`p6$?6pF7LZ6-81!HKrVteR)|Emx#0D{;NU$1!NWRmz{nRpVPe&PfhbNEXM0R{M-eIuYde;-jX5OmB~alUH=tua=eF} z7Ue*uHZ=}cHa#x{0WpvL247Ou5@1b=27xDMC&s8>Kt=xZ;sE6Rl^gxS)DZUw$Z@|0 z{qc-*bcmrHX<~-E{9pNTl$9=$M^Z!ySQ^_9{?Jb*r49Wn@b)E|p4W@gKq~^ugO<=# zMB6?^XO5#Bxh}7ifDPMSwg*nYKE_aY7@A2c!}HvH_L>@iNegRt~;{3Sc{rix7r~PVjp%V?4Y4+duj&AN~ zG5_JTox9@DF*9D`T>wBw?nL1J^lphRqzZiAF555gTTyZ}%WRW+eU9^*yI~k!i)NhI zs&bp-vXGZWHsfmcyEaJz7m#uEIQ_EqYenGBE@Hm8%k890K;a~|T`_ZQ$DT$V-j%0) zXoFU3Sb7BJGW>|%D?S_Q$-JhjpgI*8N1dlW)-iCj8Si28Q7VV7p_6crZ}U2p6zYnk z=@M3yEgipgqBgE9gAD^mqFV4P_}}_D8;3(BqEL75QhPY!&0r{-xg%*esr%1o8k-4g>x?j3M zn&#f(+N^2$IZhw=I~A$!kq2OG(?nV7`=#cH(zO9iiw5T^&tS?Eo1<8&Ga}_q4j-o; zJrSU1@9PDHWm)Bp4%Qh7$}NZTcLR&Q`k_k?IDf&|+m77qK)-7UX&|yR!X2dDL8!tT zY9q>M%Gc)CF_8M%si^QEe8`MUhdb@?8+tep+VX>7_ru<$D77Sgcx z<#Im`;`yhWT-&PhHQYH4k^7%qb7u=|qw$|%E8VblT-EMBa2jdHhhej1|7j^9TIlM( zbkj1T6no*XEE8P!iE=DZ^^dp?ZlW}0v;LEL4(a`Z`B&O8v|jlSfUFu_P0;^IJc+3P zM`M3cP4NF$ap>@LaHmJeir#*U`ZTOME|4Wh?s@$e6%G{*)$@5K_0j=EqX4xL^Bvw4 zSq?`8w>|``fx6_>e8*C34eQ?%zhWF^~Pz;uVd-rHW9B zeubdBB2BLheVdy*ukJ@ZR6K+51k%P#52k%#vJXUgBMEUvUCkOOPJG;3DElEuTcZlh z5e)1k4dHd-OmsV5AxYGbqP}CzIRugcALm=u;d5K`JIE8ELZZ~y z6owpDtAXM+q2@;I1g)PrlC@rfqT6Ed#wYEg>VGc5Pi#kbz~#9mlL!$VIqtQ%YaKgH z-Mi|mMDNLG3x$z8?P(BsD@%w-IO-(&(T5f@QH-Y>xM-2)s{v^9fn1p=y~b9r;-w;V z!gRQDjHhtyQ@52}?(@x>3i9~nA6tB^LND8ACJE|kd*=BHFLpFAj;6`~NhMuYQm1c)yN~+w2t$vj>K6 zW~<10ja*Ry(N!;n(3UR`g`&42r%|OfMWo)ci6)>2((i)=O9xv$`7nD|ZJ$>WBLJ?9 z+$R~M9Rds#LC%3ES06letS?btV^Wk5_!tgv3ZFzc~_%VOz%Z}8aU-+>!B}p=u+3u*)?PZ_jFJ{b=3{J zz`gl1dl_sbWklD?^v@e;3ml7L6mC2|X8i*a@4bafhjLS^bAz#%YE9Vl8kA1zfVQg=( z@Vh%}k~L;K6PyYRyo0s%XL-Fy97qm5+FuY35VVgZtxU^4^y*;y>8TI2(4Efk=YhNw z0ZYs;QOh$>M9^Ekl2$2sg2MCd=YrYp#F0k;hFI*)4Ng}OP-s2rG;M@&#V?#({W}3B ziM=AL5Z-+Ko0XVTv(V&Tw}D79O2aqMW0;%DHH#EIKp|VBFH|e^d);PwFg|X@AGVcr z{bG%AD0+Lp&r>9gZ&*~0oIm$2ohJ-wuMgxh2&;Zo$ACNa>?B#;zTr}YH&SjtvP+tZFao;yo0ULzk68 zot=2|P@1vi2e7mBXW)V8Su6~}uQKd|fBIcjfrFmP3lN$C6=s^868k*rx@4>#H`MH1|V3|*#y;tc{$?xl*; zl0syFoq|tUfHe`VitB7MOIvtYATe5;rpJl2CzINfA#J%(qTvQj=LD`2GO4Uw4h6l= z2xb+&{qyM4hxv3tnkeeiTIPDbGr0|8K(cZtW;>HRghDB=I6lkL!!r#4djeLYd`dK>N)9PzL#&;n4r0213(UY{Yt z7il?1$?W|bIf4N?vL}PfZG~hu-M!5$0y#?#FPXgbX*^Z?OENu9)BN{JRM&?a9N*~3 z@`KUuX$CcmGhs?zK{cm%wWKTOTNX^QthJ^9kR;iAA~qRnrr5(i-!2;D`miqgcp^~t zX(H=yv7u!n=Oi|OI<+PVqSwgy@@#x3GBPoC!XfJ^Q>ky!XZda=#7I-2uCX-m)%ui1 zx2ByShTC$t_S`tJab;7cyqzx^u(#H@@*;^_)UgEsQ8O&<(!K+rj0p5k8^`pWQ~}HD z3s=7hR2vC$6%bX7B)s@a9IOure0Kb$p>=bIcUOdMLAs(U``sapUX0rgIqs+3X_vS@ z9Pqai;y2#B{xJTJ$lr2G?Opy=t#W!ngMK)qMJD|Dr*yPbBx;ubEvb0z9W(oBnTv22 Ux!TjW&N^}Tw&|_P8%`1b2M}vu@&Et; literal 0 HcmV?d00001 diff --git a/titles/chuni/read.py b/titles/chuni/read.py index 15557d4..12b0d9e 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -2,6 +2,7 @@ from typing import Optional from os import walk, path import xml.etree.ElementTree as ET from read import BaseReader +from PIL import Image from core.config import CoreConfig from titles.chuni.database import ChuniData @@ -164,6 +165,16 @@ class ChuniReader(BaseReader): for jaketFile in xml_root.findall("jaketFile"): # nice typo, SEGA jacket_path = jaketFile.find("path").text + # Convert the image to png and save it for use in the frontend + jacket_filename_src = f"{root}/{dir}/{jacket_path}" + (pre, ext) = path.splitext(jacket_path) + jacket_filename_dst = f"titles/chuni/img/jacket/{pre}.png" + if path.exists(jacket_filename_src) and not path.exists(jacket_filename_dst): + try: + im = Image.open(jacket_filename_src) + im.save(jacket_filename_dst) + except Exception: + self.logger.warning(f"Failed to convert {jacket_path} to png") for fumens in xml_root.findall("fumens"): for MusicFumenData in fumens.findall("MusicFumenData"): diff --git a/titles/chuni/schema/__init__.py b/titles/chuni/schema/__init__.py index 51d950b..cf1f5f2 100644 --- a/titles/chuni/schema/__init__.py +++ b/titles/chuni/schema/__init__.py @@ -1,6 +1,6 @@ from titles.chuni.schema.profile import ChuniProfileData -from titles.chuni.schema.score import ChuniScoreData +from titles.chuni.schema.score import ChuniScoreData, ChuniRomVersion from titles.chuni.schema.item import ChuniItemData from titles.chuni.schema.static import ChuniStaticData -__all__ = ["ChuniProfileData", "ChuniScoreData", "ChuniItemData", "ChuniStaticData"] +__all__ = ["ChuniProfileData", "ChuniScoreData", "ChuniRomVersion", "ChuniItemData", "ChuniStaticData"] diff --git a/titles/chuni/schema/item.py b/titles/chuni/schema/item.py index 9ce2c53..92910da 100644 --- a/titles/chuni/schema/item.py +++ b/titles/chuni/schema/item.py @@ -359,6 +359,25 @@ class ChuniItemData(BaseData): return None return result.lastrowid + async def is_favorite( + self, user_id: int, version: int, fav_id: int, fav_kind: int = 1 + ) -> bool: + + sql = favorite.select( + and_( + favorite.c.version == version, + favorite.c.user == user_id, + favorite.c.favId == fav_id, + favorite.c.favKind == fav_kind, + ) + ) + + result = await self.execute(sql) + if result is None: + return False + + return True if len(result.all()) else False + async def get_all_favorites( self, user_id: int, version: int, fav_kind: int = 1 ) -> Optional[List[Row]]: @@ -421,6 +440,31 @@ class ChuniItemData(BaseData): return None return result.fetchone() + async def put_favorite_music(self, user_id: int, version: int, music_id: int) -> Optional[int]: + sql = insert(favorite).values(user=user_id, version=version, favId=music_id, favKind=1) + + conflict = sql.on_duplicate_key_update(user=user_id, version=version, favId=music_id, favKind=1) + + result = await self.execute(conflict) + if result is None: + return None + return result.lastrowid + + async def delete_favorite_music(self, user_id: int, version: int, music_id: int) -> Optional[int]: + sql = delete(favorite).where( + and_( + favorite.c.user==user_id, + favorite.c.version==version, + favorite.c.favId==music_id, + favorite.c.favKind==1 + ) + ) + + result = await self.execute(sql) + if result is None: + return None + return result.lastrowid + async def put_character(self, user_id: int, character_data: Dict) -> Optional[int]: character_data["user"] = user_id diff --git a/titles/chuni/schema/score.py b/titles/chuni/schema/score.py index 0d327f8..308afa8 100644 --- a/titles/chuni/schema/score.py +++ b/titles/chuni/schema/score.py @@ -8,6 +8,7 @@ from sqlalchemy.sql import func, select from sqlalchemy.dialects.mysql import insert from sqlalchemy.sql.expression import exists from core.data.schema import BaseData, metadata +from ..config import ChuniConfig course = Table( "chuni_score_course", @@ -140,6 +141,92 @@ playlog = Table( mysql_charset="utf8mb4" ) +class ChuniRomVersion(): + """ + Class used to easily compare rom version strings and map back to the internal integer version. + Used with methods that touch the playlog table. + """ + Versions = {} + def init_versions(cfg: ChuniConfig): + if len(ChuniRomVersion.Versions) > 0: + # dont bother with reinit + return + + # Build up a easily comparible list of versions. Used when deriving romVersion from the playlog + all_versions = { + 10: ChuniRomVersion("1.50.0"), + 9: ChuniRomVersion("1.45.0"), + 8: ChuniRomVersion("1.40.0"), + 7: ChuniRomVersion("1.35.0"), + 6: ChuniRomVersion("1.30.0"), + 5: ChuniRomVersion("1.25.0"), + 4: ChuniRomVersion("1.20.0"), + 3: ChuniRomVersion("1.15.0"), + 2: ChuniRomVersion("1.10.0"), + 1: ChuniRomVersion("1.05.0"), + 0: ChuniRomVersion("1.00.0") + } + + # add the versions from the config + for ver in range(11,999): + cfg_ver = cfg.version.version(ver) + if cfg_ver: + all_versions[ver] = ChuniRomVersion(cfg_ver["rom"]) + else: + break + + # sort it by version number for easy iteration + ChuniRomVersion.Versions = dict(sorted(all_versions.items())) + + def __init__(self, rom_version: str) -> None: + (major, minor, maint) = rom_version.split('.') + self.major = int(major) + self.minor = int(minor) + self.maint = int(maint) + self.version = rom_version + + def __str__(self) -> str: + return self.version + + def __eq__(self, other) -> bool: + return (self.major == other.major and + self.minor == other.minor and + self.maint == other.maint) + + def __lt__(self, other) -> bool: + return (self.major < other.major) or \ + (self.major == other.major and self.minor < other.minor) or \ + (self.major == other.major and self.minor == other.minor and self.maint < other.maint) + + def __gt__(self, other) -> bool: + return (self.major > other.major) or \ + (self.major == other.major and self.minor > other.minor) or \ + (self.major == other.major and self.minor == other.minor and self.maint > other.maint) + + def get_int_version(self) -> int: + """ + Used when displaying the playlog to walk backwards from the recorded romVersion to our internal version number. + This is effectively a workaround to avoid recording our internal version number along with the romVersion in the db at insert time. + """ + for ver,rom in ChuniRomVersion.Versions.items(): + # if the version matches exactly, great! + if self == rom: + return ver + + # If this isnt the last version, use the next as an upper bound + if ver + 1 < len(ChuniRomVersion.Versions): + if self > rom and self < ChuniRomVersion.Versions[ver + 1]: + # this version fits in the middle! It must be a revision of the version + # e.g. 2.15.00 vs 2.16.00 + return ver + else: + # this is the last version in the list. + # If its greate than this one and still the same major, this call it a match + if self.major == rom.major and self > rom: + return ver + + # Only way we get here is if it was a version that started with "0." which is def invalid + return -1 class ChuniScoreData(BaseData): async def get_courses(self, aime_id: int) -> Optional[Row]: @@ -190,45 +277,66 @@ class ChuniScoreData(BaseData): return None return result.fetchall() - async def get_playlogs_limited(self, aime_id: int, index: int, count: int) -> Optional[Row]: - sql = select(playlog).where(playlog.c.user == aime_id).order_by(playlog.c.id.desc()).limit(count).offset(index * count) + async def get_playlog_rom_versions_by_int_version(self, version: int, aime_id: int = -1) -> Optional[str]: + # Get a set of all romVersion values present + sql = select([playlog.c.romVersion]) + if aime_id != -1: + # limit results to a specific user + sql = sql.where(playlog.c.user == aime_id) + sql = sql.distinct() result = await self.execute(sql) if result is None: - self.logger.warning(f" aime_id {aime_id} has no playlog ") + return None + record_versions = result.fetchall() + + # for each romVersion recorded, check if it maps back the current version we are operating on + matching_rom_versions = [] + for v in record_versions: + if ChuniRomVersion(v[0]).get_int_version() == version: + matching_rom_versions += [v[0]] + + self.logger.debug(f"romVersions {matching_rom_versions} map to version {version}") + return matching_rom_versions + + async def get_playlogs_limited(self, aime_id: int, version: int, index: int, count: int) -> Optional[Row]: + # Get a list of all the recorded romVersions in the playlog + # for this user that map to the given version. + rom_versions = await self.get_playlog_rom_versions_by_int_version(version, aime_id) + if rom_versions is None: + return None + + # Query results that have the matching romVersions + sql = select(playlog).where((playlog.c.user == aime_id) & (playlog.c.romVersion.in_(rom_versions))).order_by(playlog.c.id.desc()).limit(count).offset(index * count) + + result = await self.execute(sql) + if result is None: + self.logger.info(f" aime_id {aime_id} has no playlog for version {version}") return None return result.fetchall() - async def get_user_playlogs_count(self, aime_id: int) -> Optional[Row]: - sql = select(func.count()).where(playlog.c.user == aime_id) + async def get_user_playlogs_count(self, aime_id: int, version: int) -> Optional[Row]: + # Get a list of all the recorded romVersions in the playlog + # for this user that map to the given version. + rom_versions = await self.get_playlog_rom_versions_by_int_version(version, aime_id) + if rom_versions is None: + return None + + # Query results that have the matching romVersions + sql = select(func.count()).where((playlog.c.user == aime_id) & (playlog.c.romVersion.in_(rom_versions))) + result = await self.execute(sql) if result is None: - self.logger.warning(f" aime_id {aime_id} has no playlog ") - return None + self.logger.info(f" aime_id {aime_id} has no playlog for version {version}") + return 0 return result.scalar() async def put_playlog(self, aime_id: int, playlog_data: Dict, version: int) -> Optional[int]: - # Calculate the ROM version that should be inserted into the DB, based on the version of the ggame being inserted - # We only need from Version 10 (Plost) and back, as newer versions include romVersion in their upsert - # This matters both for gameRankings, as well as a future DB update to keep version data separate - romVer = { - 10: "1.50.0", - 9: "1.45.0", - 8: "1.40.0", - 7: "1.35.0", - 6: "1.30.0", - 5: "1.25.0", - 4: "1.20.0", - 3: "1.15.0", - 2: "1.10.0", - 1: "1.05.0", - 0: "1.00.0" - } - playlog_data["user"] = aime_id playlog_data = self.fix_bools(playlog_data) + # If the romVersion is not in the data (Version 10 and earlier), look it up from our internal mapping if "romVersion" not in playlog_data: - playlog_data["romVersion"] = romVer.get(version, "1.00.0") + playlog_data["romVersion"] = ChuniRomVersion.Versions[version] sql = insert(playlog).values(**playlog_data) @@ -238,27 +346,13 @@ class ChuniScoreData(BaseData): return result.lastrowid async def get_rankings(self, version: int) -> Optional[List[Dict]]: - # Calculates the ROM version that should be fetched for rankings, based on the game version being retrieved - # This prevents tracks that are not accessible in your version from counting towards the 10 results - romVer = { - 15: "2.20%", - 14: "2.15%", - 13: "2.10%", - 12: "2.05%", - 11: "2.00%", - 10: "1.50%", - 9: "1.45%", - 8: "1.40%", - 7: "1.35%", - 6: "1.30%", - 5: "1.25%", - 4: "1.20%", - 3: "1.15%", - 2: "1.10%", - 1: "1.05%", - 0: "1.00%" - } - sql = select([playlog.c.musicId.label('id'), func.count(playlog.c.musicId).label('point')]).where((playlog.c.level != 4) & (playlog.c.romVersion.like(romVer.get(version, "%")))).group_by(playlog.c.musicId).order_by(func.count(playlog.c.musicId).desc()).limit(10) + # Get a list of all the recorded romVersions in the playlog for the given version + rom_versions = await self.get_playlog_rom_versions_by_int_version(version) + if rom_versions is None: + return None + + # Query results that have the matching romVersions + sql = select([playlog.c.musicId.label('id'), func.count(playlog.c.musicId).label('point')]).where((playlog.c.level != 4) & (playlog.c.romVersion.in_(rom_versions))).group_by(playlog.c.musicId).order_by(func.count(playlog.c.musicId).desc()).limit(10) result = await self.execute(sql) if result is None: diff --git a/titles/chuni/schema/static.py b/titles/chuni/schema/static.py index ed67b5d..5c96812 100644 --- a/titles/chuni/schema/static.py +++ b/titles/chuni/schema/static.py @@ -454,7 +454,7 @@ class ChuniStaticData(BaseData): return result.fetchone() async def get_song(self, music_id: int) -> Optional[Row]: - sql = music.select(music.c.id == music_id) + sql = music.select(music.c.songId == music_id) result = await self.execute(sql) if result is None: diff --git a/titles/chuni/templates/chuni_favorites.jinja b/titles/chuni/templates/chuni_favorites.jinja new file mode 100644 index 0000000..a386f6a --- /dev/null +++ b/titles/chuni/templates/chuni_favorites.jinja @@ -0,0 +1,55 @@ +{% extends "core/templates/index.jinja" %} +{% block content %} + +
+ {% include 'titles/chuni/templates/chuni_header.jinja' %} + {% if favorites_by_genre is defined and favorites_by_genre is not none %} +
+

{{ cur_version_name }}

+

Favorite Count: {{ favorites_count }}

+ {% for key, genre in favorites_by_genre.items() %} +

{{ key }}

+ {% for favorite in genre %} + +
+
+
+
+ +
+
+
{{ favorite.title }}
+
+
{{ favorite.artist }}
+

+
+ +
+
+
+
+
+ {% endfor %} + {% endfor %} +
+ {% endif %} +
+ +{% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_header.jinja b/titles/chuni/templates/chuni_header.jinja index 76acdc5..6085d14 100644 --- a/titles/chuni/templates/chuni_header.jinja +++ b/titles/chuni/templates/chuni_header.jinja @@ -4,6 +4,7 @@
  • PROFILE
  • RATING
  • RECORD
    • +
  • FAVORITES
  • MUSICS
  • USER BOX
    • @@ -17,6 +18,8 @@ $('.nav-link[href="/game/chuni/playlog"]').addClass('active'); } else if (currentPath.startsWith('/game/chuni/rating')) { $('.nav-link[href="/game/chuni/rating"]').addClass('active'); + } else if (currentPath.startsWith('/game/chuni/favorites')) { + $('.nav-link[href="/game/chuni/favorites"]').addClass('active'); } else if (currentPath.startsWith('/game/chuni/musics')) { $('.nav-link[href="/game/chuni/musics"]').addClass('active'); } diff --git a/titles/chuni/templates/chuni_playlog.jinja b/titles/chuni/templates/chuni_playlog.jinja index fd30746..8e035f3 100644 --- a/titles/chuni/templates/chuni_playlog.jinja +++ b/titles/chuni/templates/chuni_playlog.jinja @@ -7,25 +7,36 @@ {% include 'titles/chuni/templates/chuni_header.jinja' %} {% if playlog is defined and playlog is not none %}
    -

    Playlog counts: {{ playlog_count }}

    +

    {{ cur_version_name }}

    +

    Playlog Count: {{ playlog_count }}

    {% set rankName = ['D', 'C', 'B', 'BB', 'BBB', 'A', 'AA', 'AAA', 'S', 'S+', 'SS', 'SS+', 'SSS', 'SSS+'] %} {% set difficultyName = ['normal', 'hard', 'expert', 'master', 'ultimate'] %} {% for record in playlog %} +
    -
    +
    +

    +
    +
    {{ record.title }}

    {{ record.artist }}
    -
    +
    {{ record.raw.userPlayDate }}
    TRACK {{ record.raw.track }}
    -
    +
    + +
    +

    {{ record.raw.score }}

    {{ rankName[record.raw.rank] }}

    -
    +
    - + diff --git a/titles/chuni/templates/css/chuni_style.css b/titles/chuni/templates/css/chuni_style.css index 0900b9b..39c68b7 100644 --- a/titles/chuni/templates/css/chuni_style.css +++ b/titles/chuni/templates/css/chuni_style.css @@ -192,4 +192,21 @@ caption { 100% { transform: translateX(-100%); } +} + +.fav { + padding: 0; + padding-left: 4px; + background-color: transparent; + border: none; + cursor: pointer; +} + +.fav-set { + color: gold; +} + +.btn-fav-remove { + padding:10px; + width:100%; } \ No newline at end of file From ed5e7dc561a0931a76a5bdca6e155b51f08b6345 Mon Sep 17 00:00:00 2001 From: daydensteve Date: Wed, 25 Sep 2024 15:21:30 +0000 Subject: [PATCH 029/130] [chuni] Added truncation to long Title and Artist Name values on import (#178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I noticed the importer failing to import music 523 (Niji-iro no Flügel) from an omni pack due to the artist name being crazy long. To address this, I added truncation to max column value length for both the Title and Artist Name values. Considered doing this for the other 3 string fields as well but I can't imagine those ever being problematic. Import now succeeds with a warning generated about the truncation occurring Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/178 Co-authored-by: daydensteve Co-committed-by: daydensteve --- titles/chuni/read.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/titles/chuni/read.py b/titles/chuni/read.py index 12b0d9e..eebdf8b 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -7,6 +7,7 @@ from PIL import Image from core.config import CoreConfig from titles.chuni.database import ChuniData from titles.chuni.const import ChuniConstants +from titles.chuni.schema.static import music as MusicTable class ChuniReader(BaseReader): @@ -144,6 +145,9 @@ class ChuniReader(BaseReader): self.logger.warning(f"Failed to insert event {id}") async def read_music(self, music_dir: str, we_diff: str = "4") -> None: + max_title_len = MusicTable.columns["title"].type.length + max_artist_len = MusicTable.columns["artist"].type.length + for root, dirs, files in walk(music_dir): for dir in dirs: if path.exists(f"{root}/{dir}/Music.xml"): @@ -154,9 +158,15 @@ class ChuniReader(BaseReader): for name in xml_root.findall("name"): song_id = name.find("id").text title = name.find("str").text + if len(title) > max_title_len: + self.logger.warning(f"Truncating music {song_id} song title") + title = title[:max_title_len] for artistName in xml_root.findall("artistName"): artist = artistName.find("str").text + if len(artist) > max_artist_len: + self.logger.warning(f"Truncating music {song_id} artist name") + artist = artist[:max_artist_len] for genreNames in xml_root.findall("genreNames"): for list_ in genreNames.findall("list"): From 3843ac6eb14b130cb8819c318c41e110344906ee Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Thu, 3 Oct 2024 19:32:17 +0000 Subject: [PATCH 030/130] mai2: calc GetGameRanking result --- titles/mai2/base.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b041028..9d85857 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -1,3 +1,4 @@ +import pymysql from datetime import datetime, timedelta from typing import Any, Dict, List import logging @@ -76,7 +77,40 @@ class Mai2Base: } async def handle_get_game_ranking_api_request(self, data: Dict) -> Dict: - return {"length": 0, "gameRankingList": []} + conn = pymysql.connect( + host=self.core_config.database.host, + port=self.core_config.database.port, + user=self.core_config.database.username, + password=self.core_config.database.password, + database=self.core_config.database.name, + charset='utf8mb4' + ) + try: + cursor = conn.cursor() + + query = """ + SELECT musicid AS id, COUNT(*) AS point + FROM mai2_playlog + GROUP BY musicid + ORDER BY point DESC + LIMIT 100 + """ + cursor.execute(query) + + results = cursor.fetchall() + ranking_list = [{"id": row[0], "point": row[1], "userName": ""} for row in results] + output = { + "type": 1, + "gameRankingList": ranking_list, + "gameRankingInstantList": None + } + + cursor.close() + conn.close() + return output + + except Exception as e: + return {'length': 0, 'gameRankingList': []} async def handle_get_game_tournament_info_api_request(self, data: Dict) -> Dict: # TODO: Tournament support From 58ae491a8ccd76cae5682b150b2da89bdc3b7295 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Thu, 3 Oct 2024 19:47:36 +0000 Subject: [PATCH 031/130] add pymysql to requirements.txt --- requirements.txt | Bin 265 -> 273 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index fe5b4efd44150f88955d20cd3f7d4df314a8990c..72d984475dad6a8a90de778d14bfec8e8683cb64 100644 GIT binary patch delta 16 XcmeBVn#i<)laYg~pfb0zxG)C*CY=Qe delta 7 OcmbQp)XB7glMw(2;sMzJ From 0cef797a8a74c6a895fbfaab8035e0bd57a6b65c Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 6 Oct 2024 03:47:10 -0400 Subject: [PATCH 032/130] mai2: rework photo uploads, relates to #67 --- .../versions/d8cd1fa04c2a_mai2_add_photos.py | 38 +++++ titles/mai2/base.py | 61 ++++---- titles/mai2/frontend.py | 130 +++++++++++++++++- titles/mai2/schema/profile.py | 54 +++++++- titles/mai2/templates/mai2_header.jinja | 3 + titles/mai2/templates/mai2_photos.jinja | 28 ++++ 6 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py create mode 100644 titles/mai2/templates/mai2_photos.jinja diff --git a/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py new file mode 100644 index 0000000..312a127 --- /dev/null +++ b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py @@ -0,0 +1,38 @@ +"""mai2_add_photos + +Revision ID: d8cd1fa04c2a +Revises: 54a84103b84e +Create Date: 2024-10-06 03:09:15.959817 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'd8cd1fa04c2a' +down_revision = '54a84103b84e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mai2_user_photo', + sa.Column('id', sa.VARCHAR(length=36), nullable=False), + sa.Column('user', sa.Integer(), nullable=False), + sa.Column('playlog_num', sa.INTEGER(), nullable=False), + sa.Column('track_num', sa.INTEGER(), nullable=False), + sa.Column('when_upload', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user'], ['aime_user.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user', 'playlog_num', 'track_num', name='mai2_user_photo_uk'), + mysql_charset='utf8mb4' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('mai2_user_photo') + # ### end Alembic commands ### diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b041028..cb74206 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, List import logging from base64 import b64decode -from os import path, stat, remove +from os import path, stat, remove, mkdir, access, W_OK from PIL import ImageFile from random import randint @@ -866,46 +866,33 @@ class Mai2Base: self.logger.warning(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}" + photo_data = await self.data.profile.get_user_photo_by_user_playlog_track(user_id, playlog_id, track_num) + + if not photo_data: + photo_id = await self.data.profile.put_user_photo(user_id, playlog_id, track_num) + else: + photo_id = photo_data['id'] - if not path.exists(f"{out_name}.bin") and div_num != 0: - self.logger.warning(f"Out of order photo upload (div_num {div_num})") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if path.exists(f"{out_name}.bin") and div_num == 0: - self.logger.warning(f"Duplicate file upload") + out_folder = f"{self.game_config.uploads.photos_dir}/{photo_id}" + out_file = f"{out_folder}/{div_num}_{div_len - 1}.bin" + + if not path.exists(out_folder): + mkdir(out_folder) + + if not access(out_folder, W_OK): + self.logger.error(f"Cannot access {out_folder}") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - elif path.exists(f"{out_name}.bin"): - fstats = stat(f"{out_name}.bin") - if fstats.st_size != 10240 * div_num: - self.logger.warning(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)") + if path.exists(out_file): + self.logger.warning(f"Photo chunk {out_file} already exists, skipping") + + else: + with open(out_file, "wb") as f: + written = f.write(photo_chunk) + + if written != len(photo_chunk): + self.logger.error(f"Writing {out_file} failed! Wrote {written} bytes, expected {photo_chunk} bytes") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - try: - with open(f"{out_name}.bin", "ab") as f: - f.write(photo_chunk) - - except Exception: - self.logger.error(f"Failed writing to {out_name}.bin") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if div_num + 1 == div_len and path.exists(f"{out_name}.bin"): - try: - p = ImageFile.Parser() - with open(f"{out_name}.bin", "rb") as f: - p.feed(f.read()) - - im = p.close() - im.save(f"{out_name}.jpeg") - except Exception: - self.logger.error(f"File {out_name}.bin failed image validation") - - try: - remove(f"{out_name}.bin") - - except Exception: - self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually") return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'} diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index 976e2c4..f1f961a 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -1,11 +1,14 @@ from typing import List from starlette.routing import Route, Mount from starlette.requests import Request -from starlette.responses import Response, RedirectResponse -from os import path +from starlette.responses import Response, RedirectResponse, FileResponse +from os import path, walk, remove import yaml import jinja2 -from datetime import datetime +from datetime import datetime, timedelta +from PIL import ImageFile +import re +import shutil from core.frontend import FE_Base, UserSession, PermissionOffset from core.config import CoreConfig @@ -31,7 +34,8 @@ class Mai2Frontend(FE_Base): Route("/", self.render_GET, methods=['GET']), Mount("/playlog", routes=[ Route("/", self.render_GET_playlog, methods=['GET']), - Route("/{index}", self.render_GET_playlog, methods=['GET']), + Route("/{index:int}", self.render_GET_playlog, methods=['GET']), + Route("/photos", self.render_GET_photos, methods=['GET']), ]), Mount("/events", routes=[ Route("/", self.render_events, methods=['GET']), @@ -41,6 +45,7 @@ class Mai2Frontend(FE_Base): ]), Route("/update.name", self.update_name, methods=['POST']), Route("/version.change", self.version_change, methods=['POST']), + Route("/photo/{photo_id}", self.get_photo, methods=['GET']), ] async def render_GET(self, request: Request) -> bytes: @@ -140,6 +145,50 @@ class Mai2Frontend(FE_Base): else: return RedirectResponse("/gate/", 303) + async def render_GET_photos(self, request: Request) -> bytes: + template = self.environment.get_template( + "titles/mai2/templates/mai2_photos.jinja" + ) + usr_sesh = self.validate_session(request) + if not usr_sesh: + usr_sesh = UserSession() + + if usr_sesh.user_id > 0: + if usr_sesh.maimai_version < 0: + return RedirectResponse("/game/mai2/", 303) + + photos = await self.data.profile.get_user_photos_by_user(usr_sesh.user_id) + + photos_fixed = [] + for photo in photos: + if datetime.now().timestamp() > (photo['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo['id']) + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg"): + remove(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg") + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}"): + shutil.rmtree(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}") + + continue + + photos_fixed.append({ + "id": photo['id'], + "playlog_num": photo['playlog_num'], + "track_num": photo['track_num'], + "when_upload": photo['when_upload'], + }) + + return Response(template.render( + title=f"{self.core_config.server.name} | {self.nav_name}", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + photos=photos_fixed, + expire_days=7, + ), media_type="text/html; charset=utf-8") + else: + return RedirectResponse("/gate/", 303) + async def update_name(self, request: Request) -> bytes: usr_sesh = self.validate_session(request) if not usr_sesh: @@ -299,3 +348,76 @@ class Mai2Frontend(FE_Base): await self.data.static.update_event_by_id(int(event_id), new_enabled, new_start_date) return RedirectResponse("/game/mai2/events/?s=1", 303) + + async def get_photo(self, request: Request) -> RedirectResponse: + usr_sesh = self.validate_session(request) + if not usr_sesh: + return RedirectResponse("/gate/", 303) + + photo_jpeg = request.path_params.get("photo_id", None) + if not photo_jpeg: + return Response(status_code=400) + + matcher = re.match(r"^([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}).jpeg$", photo_jpeg) + if not matcher: + return Response(status_code=400) + + photo_id = matcher.groups()[0] + photo_info = await self.data.profile.get_user_photo_by_id(photo_id) + if not photo_info: + return Response(status_code=404) + + if photo_info["user"] != usr_sesh.user_id: + return Response(status_code=403) + + out_folder = f"{self.game_cfg.uploads.photos_dir}/{photo_id}" + + if datetime.now().timestamp() > (photo_info['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo_info['id']) + if path.exists(f"{out_folder}.jpeg"): + remove(f"{out_folder}.jpeg") + + if path.exists(f"{out_folder}"): + shutil.rmtree(out_folder) + + return Response(status_code=404) + + if path.exists(f"{out_folder}"): + print("path exists") + max_idx = 0 + p = ImageFile.Parser() + for _, _, files in walk("out_folder"): + if not files: + break + + matcher = re.match("^(\d+)_(\d+)$", files[0]) + if not matcher: + break + + max_idx = int(matcher.groups()[1]) + + if max_idx + 1 != len(files): + self.logger.error(f"Expected {max_idx + 1} files, found {len(files)}") + max_idx = 0 + break + + if max_idx == 0: + return Response(status_code=500) + + for i in range(max_idx + 1): + with open(f"{out_folder}/{i}_{max_idx}", "rb") as f: + p.feed(f.read()) + try: + im = p.close() + im.save(f"{out_folder}.jpeg") + + except Exception as e: + self.logger.error(f"{photo_id} failed PIL validation! - {e}") + + shutil.rmtree(out_folder) + + if path.exists(f"{out_folder}.jpeg"): + print(f"{out_folder}.jpeg exists") + return FileResponse(f"{out_folder}.jpeg") + + return Response(status_code=404) diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index 3ff85d2..ede0adf 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -1,9 +1,10 @@ from core.data.schema import BaseData, metadata from titles.mai2.const import Mai2Constants +from uuid import uuid4 from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -529,6 +530,22 @@ intimacy = Table( mysql_charset="utf8mb4", ) +photo = Table( # end-of-credit memorial photos, NOT user portraits + "mai2_user_photo", + metadata, + Column("id", VARCHAR(36), primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("playlog_num", INTEGER, nullable=False), + Column("track_num", INTEGER, nullable=False), + Column("when_upload", TIMESTAMP, nullable=False, server_default=func.now()), + UniqueConstraint("user", "playlog_num", "track_num", name="mai2_user_photo_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -945,6 +962,41 @@ class Mai2ProfileData(BaseData): self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def put_user_photo(self, user_id: int, playlog_num: int, track_num: int) -> Optional[str]: + photo_id = str(uuid4()) + sql = insert(photo).values( + id = photo_id, + user = user_id, + playlog_num = playlog_num, + track_num = track_num, + ) + + conflict = sql.on_duplicate_key_update(user = user_id) + + result = await self.execute(conflict) + if result: + return photo_id + + async def get_user_photo_by_id(self, photo_id: str) -> Optional[Row]: + result = await self.execute(photo.select(photo.c.id.like(photo_id))) + if result: + return result.fetchone() + + async def get_user_photo_by_user_playlog_track(self, user_id: int, playlog_num: int, track_num: int) -> Optional[Row]: + result = await self.execute(photo.select(and_(and_(photo.c.user == user_id, photo.c.playlog_num == playlog_num), photo.c.track_num == track_num))) + if result: + return result.fetchone() + + async def get_user_photos_by_user(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(photo.select(photo.c.user == user_id)) + if result: + return result.fetchall() + + async def delete_user_photo_by_id(self, photo_id: str) -> Optional[List[Row]]: + result = await self.execute(photo.delete(photo.c.id.like(photo_id))) + if not result: + self.logger.error(f"Failed to delete photo {photo_id}") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name diff --git a/titles/mai2/templates/mai2_header.jinja b/titles/mai2/templates/mai2_header.jinja index f226fbe..7e6757f 100644 --- a/titles/mai2/templates/mai2_header.jinja +++ b/titles/mai2/templates/mai2_header.jinja @@ -3,6 +3,7 @@
    JUSTICE CRITIALJUSTICE CRITICAL {{ record.raw.judgeCritical + record.raw.judgeHeaven }}
    + + + + + + + + + + + + +
    AVATAR
    + + + + + + + + + + + + +
    Wear:
    Face:
    Head:
    Skin:
    Item:
    Front:
    Back:
    +      + +
    +
    +
    +
    + + +
    + + + +
    + {% for item in wears.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in faces.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in heads.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in skins.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in items.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in fronts.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in backs.values() %} + {{ item[ + + {% endfor %} +
    + +
    + + {% if error is defined %} + {% include "core/templates/widgets/err_banner.jinja" %} + {% endif %} +
    + + + +{% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_favorites.jinja b/titles/chuni/templates/chuni_favorites.jinja index a386f6a..9ed23c3 100644 --- a/titles/chuni/templates/chuni_favorites.jinja +++ b/titles/chuni/templates/chuni_favorites.jinja @@ -7,15 +7,10 @@ {% include 'titles/chuni/templates/chuni_header.jinja' %} {% if favorites_by_genre is defined and favorites_by_genre is not none %}
    -

    {{ cur_version_name }}

    Favorite Count: {{ favorites_count }}

    {% for key, genre in favorites_by_genre.items() %}

    {{ key }}

    {% for favorite in genre %} -
    @@ -28,7 +23,7 @@
    {{ favorite.artist }}


    - +
    @@ -51,5 +46,16 @@ } }); }); + + // Remove Favorite + function removeFavorite(musicId) { + $.post("/game/chuni/update.favorite_music_favorites", { musicId: musicId, isAdd: 0 }) + .done(function (data) { + location.reload(); + }) + .fail(function () { + alert("Failed to remove favorite."); + }); + } {% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_header.jinja b/titles/chuni/templates/chuni_header.jinja index 6085d14..56f8b39 100644 --- a/titles/chuni/templates/chuni_header.jinja +++ b/titles/chuni/templates/chuni_header.jinja @@ -1,5 +1,5 @@
    -

    Chunithm

    +

    {{ cur_version_name }}
    \ No newline at end of file diff --git a/titles/chuni/templates/chuni_index.jinja b/titles/chuni/templates/chuni_index.jinja index 1854a89..c0e22b9 100644 --- a/titles/chuni/templates/chuni_index.jinja +++ b/titles/chuni/templates/chuni_index.jinja @@ -69,9 +69,48 @@ Last Play Date: {{ profile.lastPlayDate }} + {% if cur_version >= 6 %} + + Map Icon: +
    {{ map_icons[profile.mapIconId]["name"] }}
    + + + System Voice: +
    {{ system_voices[profile.voiceId]["name"] }}
    + + {% endif %}
    + + {% if cur_version >= 6 %} + +
    +
    + +
    + {% for item in map_icons.values() %} + {{ item[ + + {% endfor %} +
    +
    +
    + + +
    +
    + +
    + {% for item in system_voices.values() %} + {{ item[ + + {% endfor %} +
    +
    +
    + {% endif %} +
    @@ -147,4 +186,93 @@ }); } + +{% if cur_version >= 6 %} + +{% endif %} + {% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_playlog.jinja b/titles/chuni/templates/chuni_playlog.jinja index 8e035f3..cdd3f98 100644 --- a/titles/chuni/templates/chuni_playlog.jinja +++ b/titles/chuni/templates/chuni_playlog.jinja @@ -7,20 +7,15 @@ {% include 'titles/chuni/templates/chuni_header.jinja' %} {% if playlog is defined and playlog is not none %}
    -

    {{ cur_version_name }}

    Playlog Count: {{ playlog_count }}

    {% set rankName = ['D', 'C', 'B', 'BB', 'BBB', 'A', 'AA', 'AAA', 'S', 'S+', 'SS', 'SS+', 'SSS', 'SSS+'] %} {% set difficultyName = ['normal', 'hard', 'expert', 'master', 'ultimate'] %} {% for record in playlog %} - - - -
    -

    +

    {{ '★' if record.isFav else '☆' }}

    {{ record.title }}
    @@ -191,5 +186,23 @@ } }); }); + + // Add/Remove Favorite + function updateFavorite(elementId, musicId) { + element = document.getElementById(elementId); + isAdd = 1; + if (element.classList.contains("fav-set")) + { + isAdd = 0; + } + + $.post("/game/chuni/update.favorite_music_favorites", { musicId: musicId, isAdd: isAdd }) + .done(function (data) { + location.reload(); + }) + .fail(function () { + alert("Failed to update favorite."); + }); + } {% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_userbox.jinja b/titles/chuni/templates/chuni_userbox.jinja new file mode 100644 index 0000000..bd64943 --- /dev/null +++ b/titles/chuni/templates/chuni_userbox.jinja @@ -0,0 +1,256 @@ +{% extends "core/templates/index.jinja" %} +{% block content %} + + +
    + {% include 'titles/chuni/templates/chuni_header.jinja' %} + + +
    +
    +
    +
    + + + + + + + + + + +
    USER BOX
    + + + + + +
    {{team_name}}
    + + + +
    + + + +
    + Lv. + {{ profile.level }}   {{ profile.userName }} +
    +
    + RATING +   {{ profile.playerRating/100 }} +
    + + + + +
    Nameplate:
    Trophy:
    + +
    Character:
    +      + +
    +
    +
    +
    + + +
    + + + +
    + {% for item in nameplates.values() %} + {{ item[ + + {% endfor %} +
    +
    + + + +
    + {% for item in characters.values() %} + {{ item[ + + {% endfor %} +
    + +
    + + {% if error is defined %} + {% include "core/templates/widgets/err_banner.jinja" %} + {% endif %} +
    + + + +{% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/css/chuni_style.css b/titles/chuni/templates/css/chuni_style.css index 39c68b7..18ce6c5 100644 --- a/titles/chuni/templates/css/chuni_style.css +++ b/titles/chuni/templates/css/chuni_style.css @@ -159,6 +159,45 @@ caption { font-weight: bold; } +.rating { + font-weight: bold; + -webkit-text-stroke-width: 1px; + -webkit-text-stroke-color: black; +} + +.rating-rank0 { + color: #008000; +} + +.rating-rank1 { + color: #ffa500; +} +.rating-rank2 { + color: #ff0000; +} +.rating-rank3 { + color: #800080; +} +.rating-rank4 { + color: #cd853f; +} +.rating-rank5 { + color: #c0c0c0; +} +.rating-rank6 { + color: #ffd700; +} +.rating-rank7 { + color: #a9a9a9; +} + +.rating-rank8 { + background: linear-gradient(to right, red, yellow, lime, aqua, blue, fuchsia) 0 / 5em; + background-clip: text; + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; +} + .scrolling-text { overflow: hidden; } @@ -194,6 +233,41 @@ caption { } } +/* + Styles to support collapsible boxes (used for browsing images) +*/ +.collapsible { + background-color: #555; + cursor: pointer; + padding-bottom: 16px; + width: 100%; + border: none; + text-align: left; + outline: none; + font-family: monospace; + font-weight: bold; +} + + .collapsible:after { + content: '[+]'; + float: right; + } + +.collapsible-active:after { + content: "[-]"; +} + +.collapsible-content { + max-height: 0px; + overflow: hidden; + opacity: 0; + transition: max-height 0.2s ease-out; + background-color: #DDD; +} + +/* + Styles for favorites star in /playlog +*/ .fav { padding: 0; padding-left: 4px; @@ -206,7 +280,257 @@ caption { color: gold; } +/* + Styles for favorites in /favorites +*/ .btn-fav-remove { padding:10px; width:100%; +} + +/* + Styles for userbox configuration +*/ +.userbox { + position: absolute; +} + +.userbox-nameplate { + top: 72px; + left: 32px; +} + +.userbox-teamframe { + top: 74px; + left: 156px; +} + +.userbox-teamname { + top: 72px; + left: 254px; + padding: 8px 20px; + font-size: 22px; + text-shadow: rgba(0,0,0,0.8) 2px 2px; + color: #DDD; + width: 588px; + text-align: left; +} + +.userbox-trophy { + top: 170px; + left: 250px; + zoom: 0.70; +} + +.userbox-trophy-name { + top: 170px; + left: 250px; + padding: 8px 20px; + font-size: 28px; + font-weight: bold; + color: #333; + width: 588px; + text-align: center; +} + +.userbox-ratingframe { + top: 160px; + left: 175px; +} + +.userbox-charaframe { + top: 267px; + left: 824px; + zoom: 0.61; +} + +.userbox-chara { + top: 266px; + left: 814px; + zoom: 0.62; +} + +.userbox-name { + top: 160px; + left: 162px; + padding: 8px 20px; + font-size: 32px; + font-weight: bold; + color: #333; + text-align: left; +} + +.userbox-name-level-label { + font-size: 24px; +} + +.userbox-rating { + top: 204px; + left: 166px; + padding: 8px 20px; + font-size: 24px; + text-align: left; +} + +.userbox-rating-label { + font-size: 16px; +} + +.trophy-rank0 { + color: #111; + background-color: #DDD; +} +.trophy-rank1 { + color: #111; + background-color: #D85; +} +.trophy-rank2 { + color: #111; + background-color: #ADF; +} +.trophy-rank3 { + color: #111; + background-color: #EB3; +} +.trophy-rank4 { + color: #111; + background-color: #EB3; +} +.trophy-rank5 { + color: #111; + background-color: #FFA; +} +.trophy-rank6 { + color: #111; + background-color: #FFA; +} +.trophy-rank7 { + color: #111; + background-color: #FCF; +} +.trophy-rank8 { + color: #111; + background-color: #FCF; +} +.trophy-rank9 { + color: #111; + background-color: #07C; +} +.trophy-rank10 { + color: #111; + background-color: #7FE; +} +.trophy-rank11 { + color: #111; + background-color: #8D7; +} + +/* + Styles for scrollable divs (used for browsing images) +*/ +.scrolling-lists { + table-layout: fixed; +} + + .scrolling-lists div { + overflow: auto; + white-space: nowrap; + } + + .scrolling-lists img { + width: 128px; + } + +.scrolling-lists-lg { + table-layout: fixed; + width: 100%; +} + + .scrolling-lists-lg div { + overflow: auto; + white-space: nowrap; + padding: 4px; + } + + .scrolling-lists-lg img { + padding: 4px; + width: 128px; + } + +/* + Styles for avatar configuration +*/ +.avatar-preview { + position:absolute; + zoom:0.5; +} + +.avatar-preview-wear { + top: 280px; + left: 60px; +} + +.avatar-preview-face { + top: 262px; + left: 200px; +} + +.avatar-preview-head { + top: 130px; + left: 120px; +} + +.avatar-preview-skin-body { + top: 250px; + left: 190px; + height: 406px; + width: 256px; + object-fit: cover; + object-position: top; +} + +.avatar-preview-skin-leftfoot { + top: 625px; + left: 340px; + object-position: -84px -406px; +} + +.avatar-preview-skin-rightfoot { + top: 625px; + left: 40px; + object-position: 172px -406px; +} + +.avatar-preview-common { + top: 250px; + left: 135px; +} + +.avatar-preview-item-lefthand { + top: 180px; + left: 370px; + height: 544px; + width: 200px; + object-fit: cover; + object-position: right; +} + +.avatar-preview-item-righthand { + top: 180px; + left: 65px; + height: 544px; + width: 200px; + object-fit: cover; + object-position: left; +} + +.avatar-preview-back { + top: 140px; + left: 46px; +} + +.avatar-preview-platform { + top: 310px; + left: 55px; + zoom: 1; } \ No newline at end of file diff --git a/titles/chuni/templates/scripts/collapsibles.js b/titles/chuni/templates/scripts/collapsibles.js new file mode 100644 index 0000000..ff2b5a3 --- /dev/null +++ b/titles/chuni/templates/scripts/collapsibles.js @@ -0,0 +1,66 @@ +/// +/// Handles the collapsible behavior of each of the scrollable containers +/// +/// @note Intent is to include this file via jinja in the same +{% else %} +{% endif %} {% endblock content %} \ No newline at end of file diff --git a/titles/chuni/templates/chuni_index.jinja b/titles/chuni/templates/chuni_index.jinja index c0e22b9..417b04f 100644 --- a/titles/chuni/templates/chuni_index.jinja +++ b/titles/chuni/templates/chuni_index.jinja @@ -72,11 +72,11 @@ {% if cur_version >= 6 %} Map Icon: -
    {{ map_icons[profile.mapIconId]["name"] }}
    +
    {{ map_icons[profile.mapIconId]["name"] if map_icons|length > 0 else "Server DB needs upgraded or is not populated with necessary data" }}
    System Voice: -
    {{ system_voices[profile.voiceId]["name"] }}
    +
    {{ system_voices[profile.voiceId]["name"] if system_voices|length > 0 else "Server DB needs upgraded or is not populated with necessary data" }}
    {% endif %} diff --git a/titles/chuni/templates/chuni_userbox.jinja b/titles/chuni/templates/chuni_userbox.jinja index fbc0110..ab0f821 100644 --- a/titles/chuni/templates/chuni_userbox.jinja +++ b/titles/chuni/templates/chuni_userbox.jinja @@ -91,6 +91,12 @@ {% endif %}
    +{% if nameplates|length == 0 or characters|length == 0 %} + +{% else %} - +{% endif %} {% endblock content %} \ No newline at end of file From 8a6250bebd4ba8ce90ae7092b5d7bee58399161f Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Mon, 11 Nov 2024 21:11:33 +0800 Subject: [PATCH 054/130] Formatted log print Change log level --- titles/mai2/frontend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index 436d488..b39a37e 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -389,7 +389,7 @@ class Mai2Frontend(FE_Base): for _, _, files in walk("out_folder"): if not files: break - + matcher = re.match("^(\d+)_(\d+)$", files[0]) if not matcher: break @@ -410,6 +410,7 @@ class Mai2Frontend(FE_Base): try: im = p.close() im.save(f"{out_folder}.jpeg") + self.logger.info(f"{out_folder}.jpeg generated.") except Exception as e: self.logger.error(f"{photo_id} failed PIL validation! - {e}") From f4dff9b4c1bd09b536b9e2e3b2c638a2c973c0de Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Mon, 11 Nov 2024 21:16:19 +0800 Subject: [PATCH 055/130] fix: mai2 photos cant be merged --- titles/mai2/frontend.py | 130 ++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index b39a37e..9667eca 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -55,7 +55,7 @@ class Mai2Frontend(FE_Base): usr_sesh = self.validate_session(request) if not usr_sesh: usr_sesh = UserSession() - + incoming_ver = usr_sesh.maimai_version if usr_sesh.user_id > 0: @@ -103,10 +103,10 @@ class Mai2Frontend(FE_Base): if not path_index or int(path_index) < 1: index = 0 else: - index = int(path_index) - 1 # 0 and 1 are 1st page + index = int(path_index) - 1 # 0 and 1 are 1st page user_id = usr_sesh.user_id playlog_count = await self.data.score.get_user_playlogs_count(user_id) - if playlog_count < index * 20 : + if playlog_count < index * 20: return Response(template.render( title=f"{self.core_config.server.name} | {self.nav_name}", game_list=self.environment.globals["game_list"], @@ -116,17 +116,17 @@ class Mai2Frontend(FE_Base): playlog = await self.data.score.get_playlogs(user_id, index, 20) playlog_with_title = [] for record in playlog: - music_chart = await self.data.static.get_music_chart(usr_sesh.maimai_version, record.musicId, record.level) + music_chart = await self.data.static.get_music_chart(usr_sesh.maimai_version, record.musicId, record.level) if music_chart: - difficultyNum=music_chart.chartId - difficulty=music_chart.difficulty - artist=music_chart.artist - title=music_chart.title + difficultyNum = music_chart.chartId + difficulty = music_chart.difficulty + artist = music_chart.artist + title = music_chart.title else: - difficultyNum=0 - difficulty=0 - artist="unknown" - title="musicid: " + str(record.musicId) + difficultyNum = 0 + difficulty = 0 + artist = "unknown" + title = "musicid: " + str(record.musicId) playlog_with_title.append({ "raw": record, "title": title, @@ -156,29 +156,29 @@ class Mai2Frontend(FE_Base): if usr_sesh.user_id > 0: if usr_sesh.maimai_version < 0: return RedirectResponse("/game/mai2/", 303) - + photos = await self.data.profile.get_user_photos_by_user(usr_sesh.user_id) photos_fixed = [] for photo in photos: if datetime.now().timestamp() > (photo['when_upload'] + timedelta(days=7)).timestamp(): await self.data.profile.delete_user_photo_by_id(photo['id']) - + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg"): remove(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg") if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}"): shutil.rmtree(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}") - + continue - + photos_fixed.append({ "id": photo['id'], "playlog_num": photo['playlog_num'], "track_num": photo['track_num'], "when_upload": photo['when_upload'], }) - + return Response(template.render( title=f"{self.core_config.server.name} | {self.nav_name}", game_list=self.environment.globals["game_list"], @@ -195,7 +195,7 @@ class Mai2Frontend(FE_Base): return RedirectResponse("/gate/", 303) form_data = await request.form() - new_name: str = form_data.get("new_name") + new_name: str = form_data.get("new_name") new_name_full = "" if not new_name: @@ -204,7 +204,7 @@ class Mai2Frontend(FE_Base): if len(new_name) > 8: return RedirectResponse("/gate/?e=8", 303) - for x in new_name: # FIXME: This will let some invalid characters through atm + for x in new_name: # FIXME: This will let some invalid characters through atm o = ord(x) try: if o == 0x20: @@ -235,13 +235,13 @@ class Mai2Frontend(FE_Base): resp = RedirectResponse("/game/mai2/events/", 303) else: resp = RedirectResponse("/game/mai2/", 303) - + if usr_sesh.user_id > 0: form_data = await request.form() maimai_version = form_data.get("version") self.logger.info(f"version change to: {maimai_version}") - if(maimai_version.isdigit()): - usr_sesh.maimai_version=int(maimai_version) + if (maimai_version.isdigit()): + usr_sesh.maimai_version = int(maimai_version) encoded_sesh = self.encode_session(usr_sesh) self.logger.debug(f"Created session with JWT {encoded_sesh}") resp.set_cookie("ARTEMIS_SESH", encoded_sesh) @@ -253,20 +253,20 @@ class Mai2Frontend(FE_Base): usr_sesh = self.validate_session(request) if not usr_sesh: return RedirectResponse("/gate/", 303) - + if not self.test_perm(usr_sesh.permissions, PermissionOffset.SYSADMIN): return RedirectResponse("/game/mai2/", 303) - + template = self.environment.get_template( "titles/mai2/templates/events/mai2_events.jinja" ) - + incoming_ver = usr_sesh.maimai_version evts = [] - + if incoming_ver < 0: usr_sesh.maimai_version = Mai2Constants.VER_MAIMAI_DX - + event_list = await self.data.static.get_game_events(usr_sesh.maimai_version) self.logger.info(f"Get events for v{usr_sesh.maimai_version}") @@ -280,88 +280,88 @@ class Mai2Frontend(FE_Base): "startDate": event['startDate'].strftime("%x %X"), "enabled": "true" if event['enabled'] else "false", }) - + resp = Response(template.render( - title=f"{self.core_config.server.name} | {self.nav_name} Events", - game_list=self.environment.globals["game_list"], - sesh=vars(usr_sesh), - version_list=Mai2Constants.VERSION_STRING, - events=evts - ), media_type="text/html; charset=utf-8") - + title=f"{self.core_config.server.name} | {self.nav_name} Events", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + version_list=Mai2Constants.VERSION_STRING, + events=evts + ), media_type="text/html; charset=utf-8") + if incoming_ver < 0: encoded_sesh = self.encode_session(usr_sesh) resp.delete_cookie("ARTEMIS_SESH") resp.set_cookie("ARTEMIS_SESH", encoded_sesh) - + return resp async def render_event_edit(self, request: Request) -> Response: usr_sesh = self.validate_session(request) if not usr_sesh: return RedirectResponse("/gate/", 303) - + if not self.test_perm(usr_sesh.permissions, PermissionOffset.SYSADMIN): return RedirectResponse("/game/mai2/", 303) - + template = self.environment.get_template( "titles/mai2/templates/events/mai2_event_edit.jinja" ) - + evt_id = request.path_params.get("event_id") - + event_id = await self.data.static.get_event_by_id(evt_id) if not event_id: return RedirectResponse("/game/mai2/events/", 303) - + return Response(template.render( - title=f"{self.core_config.server.name} | {self.nav_name} Edit Event {evt_id}", - game_list=self.environment.globals["game_list"], - sesh=vars(usr_sesh), - user_id=usr_sesh.user_id, - version_list=Mai2Constants.VERSION_STRING, - cur_version=usr_sesh.maimai_version, - event=event_id._asdict() - ), media_type="text/html; charset=utf-8") + title=f"{self.core_config.server.name} | {self.nav_name} Edit Event {evt_id}", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + user_id=usr_sesh.user_id, + version_list=Mai2Constants.VERSION_STRING, + cur_version=usr_sesh.maimai_version, + event=event_id._asdict() + ), media_type="text/html; charset=utf-8") async def update_event(self, request: Request) -> RedirectResponse: usr_sesh = self.validate_session(request) if not usr_sesh: return RedirectResponse("/gate/", 303) - + if not self.test_perm(usr_sesh.permissions, PermissionOffset.SYSADMIN): return RedirectResponse("/game/mai2/", 303) - + form_data = await request.form() print(form_data) event_id: int = form_data.get("evtId", None) - new_enabled: bool = bool(form_data.get("evtEnabled", False)) + new_enabled: bool = bool(form_data.get("evtEnabled", False)) try: new_start_date: datetime = datetime.strptime(form_data.get("evtStart", None), "%Y-%m-%dT%H:%M:%S") except: new_start_date = None - + print(f"{event_id} {new_enabled} {new_start_date}") if event_id is None or new_start_date is None: return RedirectResponse("/game/mai2/events/?e=4", 303) await self.data.static.update_event_by_id(int(event_id), new_enabled, new_start_date) - + return RedirectResponse("/game/mai2/events/?s=1", 303) async def get_photo(self, request: Request) -> RedirectResponse: usr_sesh = self.validate_session(request) if not usr_sesh: return RedirectResponse("/gate/", 303) - + photo_jpeg = request.path_params.get("photo_id", None) if not photo_jpeg: return Response(status_code=400) - + matcher = re.match(r"^([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}).jpeg$", photo_jpeg) if not matcher: return Response(status_code=400) - + photo_id = matcher.groups()[0] photo_info = await self.data.profile.get_user_photo_by_id(photo_id) if not photo_info: @@ -379,18 +379,18 @@ class Mai2Frontend(FE_Base): if path.exists(f"{out_folder}"): shutil.rmtree(out_folder) - + return Response(status_code=404) if path.exists(f"{out_folder}"): self.logger.info(f"Photo Path Exist.") max_idx = 0 p = ImageFile.Parser() - for _, _, files in walk("out_folder"): + for _, _, files in walk(f"{out_folder}"): if not files: break - matcher = re.match("^(\d+)_(\d+)$", files[0]) + matcher = re.match(r"^(\d+)_(\d+)\.bin$", files[0]) if not matcher: break @@ -400,18 +400,18 @@ class Mai2Frontend(FE_Base): self.logger.error(f"Expected {max_idx + 1} files, found {len(files)}") max_idx = 0 break - + if max_idx == 0: return Response(status_code=500) - + for i in range(max_idx + 1): - with open(f"{out_folder}/{i}_{max_idx}", "rb") as f: + with open(f"{out_folder}/{i}_{max_idx}.bin", "rb") as f: p.feed(f.read()) try: im = p.close() im.save(f"{out_folder}.jpeg") self.logger.info(f"{out_folder}.jpeg generated.") - + except Exception as e: self.logger.error(f"{photo_id} failed PIL validation! - {e}") @@ -420,5 +420,5 @@ class Mai2Frontend(FE_Base): if path.exists(f"{out_folder}.jpeg"): self.logger.info(f"{out_folder}.jpeg exists") return FileResponse(f"{out_folder}.jpeg") - + return Response(status_code=404) From b7a006f7ee3890b89e23f088eece516d7a3c2bc8 Mon Sep 17 00:00:00 2001 From: Midorica Date: Tue, 12 Nov 2024 10:53:02 -0500 Subject: [PATCH 056/130] core: pushing changes regarding MySQL ssl toggle that is now mandatory --- core/config.py | 6 ++++++ core/data/database.py | 4 ++-- docs/config.md | 1 + example_config/core.yaml | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/config.py b/core/config.py index 3d7b919..dda43aa 100644 --- a/core/config.py +++ b/core/config.py @@ -175,6 +175,12 @@ class DatabaseConfig: return CoreConfig.get_config_field( self.__config, "core", "database", "protocol", default="mysql" ) + + @property + def ssl_enabled(self) -> str: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_enabled", default=False + ) @property def sha2_password(self) -> bool: diff --git a/core/data/database.py b/core/data/database.py index bd6c4f2..b4f3cc0 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -27,9 +27,9 @@ class Data: if self.config.database.sha2_password: passwd = sha256(self.config.database.password.encode()).digest() - self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4" + self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" else: - self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4" + self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" if Data.engine is None: Data.engine = create_engine(self.__url, pool_recycle=3600) diff --git a/docs/config.md b/docs/config.md index 8a482e3..f85e8e7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -26,6 +26,7 @@ - `name`: Name of the database the server should expect. Default `aime` - `port`: Port the database server is listening on. Default `3306` - `protocol`: Protocol used in the connection string, e.i `mysql` would result in `mysql://...`. Default `mysql` +- `ssl_enabled`: Enforce SSL to be used in the connection string. Default `False` - `sha2_password`: Whether or not the password in the connection string should be hashed via SHA2. Default `False` - `loglevel`: Logging level for the database. Default `info` - `memcached_host`: Host of the memcached server. Default `localhost` diff --git a/example_config/core.yaml b/example_config/core.yaml index daf18fc..0f047f0 100644 --- a/example_config/core.yaml +++ b/example_config/core.yaml @@ -27,6 +27,7 @@ database: name: "aime" port: 3306 protocol: "mysql" + ssl_enabled: False sha2_password: False loglevel: "info" enable_memcached: True From bc7524c8fcfaf435ff214de6cbbd6033f3f6a332 Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 14 Nov 2024 12:36:22 +0700 Subject: [PATCH 057/130] fix: make database async --- core/data/alembic/env.py | 68 +++++++++++----- core/data/database.py | 156 ++++++++++++++++++++++++------------- core/data/schema/arcade.py | 30 +++---- core/data/schema/base.py | 23 +++--- core/data/schema/card.py | 11 +-- core/data/schema/user.py | 24 +++--- core/utils.py | 104 ++++++++++++++++++++----- dbutils.py | 11 +-- read.py | 24 +++--- 9 files changed, 297 insertions(+), 154 deletions(-) diff --git a/core/data/alembic/env.py b/core/data/alembic/env.py index d532093..f2a8182 100644 --- a/core/data/alembic/env.py +++ b/core/data/alembic/env.py @@ -1,8 +1,14 @@ from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool + +import asyncio +import threading from logging.config import fileConfig +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + from core.data.schema.base import metadata # this is the Alembic Config object, which provides @@ -37,20 +43,29 @@ def run_migrations_offline(): script output. """ - raise Exception('Not implemented or configured!') + raise Exception("Not implemented or configured!") url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) with context.begin_transaction(): context.run_migrations() -def run_migrations_online(): - """Run migrations in 'online' mode. +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + ) - In this scenario we need to create an Engine + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """In this scenario we need to create an Engine and associate a connection with the context. """ @@ -59,21 +74,32 @@ def run_migrations_online(): for override in overrides: ini_section[override] = overrides[override] - connectable = engine_from_config( - ini_section, - prefix='sqlalchemy.', - poolclass=pool.NullPool) + connectable = async_engine_from_config( + ini_section, prefix="sqlalchemy.", poolclass=pool.NullPool + ) - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - compare_type=True, - compare_server_default=True, - ) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online(): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + # there's no event loop + asyncio.run(run_async_migrations()) + else: + # there's currently an event loop and trying to wait for a coroutine + # to finish without using `await` is pretty wormy. nested event loops + # are explicitly forbidden by asyncio. + # + # take the easy way out, spawn it in another thread. + thread = threading.Thread(target=asyncio.run, args=(run_async_migrations(),)) + thread.start() + thread.join() - with context.begin_transaction(): - context.run_migrations() if context.is_offline_mode(): run_migrations_offline() diff --git a/core/data/database.py b/core/data/database.py index b4f3cc0..16bd67b 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -1,54 +1,65 @@ -import logging, coloredlogs -from typing import Optional -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy import create_engine -from logging.handlers import TimedRotatingFileHandler +import asyncio +import logging import os -import secrets, string -import bcrypt +import secrets +import string +import warnings from hashlib import sha256 +from logging.handlers import TimedRotatingFileHandler +from typing import ClassVar, Optional + import alembic.config -import glob +import bcrypt +import coloredlogs +import pymysql.err +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_scoped_session, + create_async_engine, +) +from sqlalchemy.orm import sessionmaker from core.config import CoreConfig -from core.data.schema import * -from core.utils import Utils +from core.data.schema import ArcadeData, BaseData, CardData, UserData, metadata +from core.utils import MISSING, Utils class Data: - engine = None - session = None - user = None - arcade = None - card = None - base = None + engine: ClassVar[AsyncEngine] = MISSING + session: ClassVar[AsyncSession] = MISSING + user: ClassVar[UserData] = MISSING + arcade: ClassVar[ArcadeData] = MISSING + card: ClassVar[CardData] = MISSING + base: ClassVar[BaseData] = MISSING + def __init__(self, cfg: CoreConfig) -> None: self.config = cfg if self.config.database.sha2_password: passwd = sha256(self.config.database.password.encode()).digest() - self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" + self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" else: - self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" + self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" - if Data.engine is None: - Data.engine = create_engine(self.__url, pool_recycle=3600) + if Data.engine is MISSING: + Data.engine = create_async_engine(self.__url, pool_recycle=3600, isolation_level="AUTOCOMMIT") self.__engine = Data.engine - if Data.session is None: - s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True) - Data.session = scoped_session(s) + if Data.session is MISSING: + s = sessionmaker(Data.engine, expire_on_commit=False, class_=AsyncSession) + Data.session = async_scoped_session(s, asyncio.current_task) - if Data.user is None: + if Data.user is MISSING: Data.user = UserData(self.config, self.session) - if Data.arcade is None: + if Data.arcade is MISSING: Data.arcade = ArcadeData(self.config, self.session) - if Data.card is None: + if Data.card is MISSING: Data.card = CardData(self.config, self.session) - if Data.base is None: + if Data.base is MISSING: Data.base = BaseData(self.config, self.session) self.logger = logging.getLogger("database") @@ -94,40 +105,73 @@ class Data: alembic.config.main(argv=alembicArgs) os.chdir(old_dir) - def create_database(self): + async def create_database(self): self.logger.info("Creating databases...") - metadata.create_all( - self.engine, - checkfirst=True, - ) - for _, mod in Utils.get_all_titles().items(): - if hasattr(mod, "database"): - mod.database(self.config) - metadata.create_all( - self.engine, - checkfirst=True, - ) + with warnings.catch_warnings(): + # SQLAlchemy will generate a nice primary key constraint name, but in + # MySQL/MariaDB the constraint name is always PRIMARY. Every time a + # custom primary key name is generated, a warning is emitted from pymysql, + # which we don't care about. Other warnings may be helpful though, don't + # suppress everything. + warnings.filterwarnings( + action="ignore", + message=r"Name '(.+)' ignored for PRIMARY key\.", + category=pymysql.err.Warning, + ) - # Stamp the end revision as if alembic had created it, so it can take off after this. - self.__alembic_cmd( - "stamp", - "head", - ) + async with self.engine.begin() as conn: + await conn.run_sync(metadata.create_all, checkfirst=True) - def schema_upgrade(self, ver: str = None): - self.__alembic_cmd( - "upgrade", - "head" if not ver else ver, - ) + for _, mod in Utils.get_all_titles().items(): + if hasattr(mod, "database"): + mod.database(self.config) + + await conn.run_sync(metadata.create_all, checkfirst=True) + + # Stamp the end revision as if alembic had created it, so it can take off after this. + self.__alembic_cmd( + "stamp", + "head", + ) + + def schema_upgrade(self, ver: Optional[str] = None): + with warnings.catch_warnings(): + # SQLAlchemy will generate a nice primary key constraint name, but in + # MySQL/MariaDB the constraint name is always PRIMARY. Every time a + # custom primary key name is generated, a warning is emitted from pymysql, + # which we don't care about. Other warnings may be helpful though, don't + # suppress everything. + warnings.filterwarnings( + action="ignore", + message=r"Name '(.+)' ignored for PRIMARY key\.", + category=pymysql.err.Warning, + ) + + self.__alembic_cmd( + "upgrade", + "head" if not ver else ver, + ) def schema_downgrade(self, ver: str): - self.__alembic_cmd( - "downgrade", - ver, - ) + with warnings.catch_warnings(): + # SQLAlchemy will generate a nice primary key constraint name, but in + # MySQL/MariaDB the constraint name is always PRIMARY. Every time a + # custom primary key name is generated, a warning is emitted from pymysql, + # which we don't care about. Other warnings may be helpful though, don't + # suppress everything. + warnings.filterwarnings( + action="ignore", + message=r"Name '(.+)' ignored for PRIMARY key\.", + category=pymysql.err.Warning, + ) - async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None: + self.__alembic_cmd( + "downgrade", + ver, + ) + + async def create_owner(self, email: Optional[str] = None, code: str = "00000000000000000000") -> None: pw = "".join( secrets.choice(string.ascii_letters + string.digits) for i in range(20) ) @@ -150,12 +194,12 @@ class Data: async def migrate(self) -> None: exist = await self.base.execute("SELECT * FROM alembic_version") if exist is not None: - self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!") + self.logger.warning("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!") return self.logger.info("Upgrading to latest with legacy system") if not await self.legacy_upgrade(): - self.logger.warn("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!") + self.logger.warning("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!") return self.logger.info("Done") diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index 5b570a1..653fe7c 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -1,16 +1,16 @@ -from typing import Optional, Dict, List -from sqlalchemy import Table, Column, and_, or_ -from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint -from sqlalchemy.types import Integer, String, Boolean, JSON -from sqlalchemy.sql import func, select +import re +from typing import List, Optional + +from sqlalchemy import Column, Table, and_, or_ from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row -import re +from sqlalchemy.sql import func, select +from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint +from sqlalchemy.types import JSON, Boolean, Integer, String from core.data.schema.base import BaseData, metadata -from core.const import * -arcade = Table( +arcade: Table = Table( "arcade", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -26,7 +26,7 @@ arcade = Table( mysql_charset="utf8mb4", ) -machine = Table( +machine: Table = Table( "machine", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -47,7 +47,7 @@ machine = Table( mysql_charset="utf8mb4", ) -arcade_owner = Table( +arcade_owner: Table = Table( "arcade_owner", metadata, Column( @@ -69,7 +69,7 @@ arcade_owner = Table( class ArcadeData(BaseData): - async def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]: + async def get_machine(self, serial: Optional[str] = None, id: Optional[int] = None) -> Optional[Row]: if serial is not None: serial = serial.replace("-", "") if len(serial) == 11: @@ -98,8 +98,8 @@ class ArcadeData(BaseData): self, arcade_id: int, serial: str = "", - board: str = None, - game: str = None, + board: Optional[str] = None, + game: Optional[str] = None, is_cab: bool = False, ) -> Optional[int]: if not arcade_id: @@ -150,8 +150,8 @@ class ArcadeData(BaseData): async def create_arcade( self, - name: str = None, - nickname: str = None, + name: Optional[str] = None, + nickname: Optional[str] = None, country: str = "JPN", country_id: int = 1, state: str = "", diff --git a/core/data/schema/base.py b/core/data/schema/base.py index d74198b..cb44272 100644 --- a/core/data/schema/base.py +++ b/core/data/schema/base.py @@ -1,22 +1,23 @@ +import asyncio import json import logging from random import randrange -from typing import Any, Optional, Dict, List +from typing import Any, Dict, List, Optional + +from sqlalchemy import Column, MetaData, Table from sqlalchemy.engine import Row from sqlalchemy.engine.cursor import CursorResult -from sqlalchemy.engine.base import Connection -from sqlalchemy.sql import text, func, select from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy import MetaData, Table, Column -from sqlalchemy.types import Integer, String, TIMESTAMP, JSON, INTEGER, TEXT +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.schema import ForeignKey -from sqlalchemy.dialects.mysql import insert +from sqlalchemy.sql import func, text +from sqlalchemy.types import INTEGER, JSON, TEXT, TIMESTAMP, Integer, String from core.config import CoreConfig metadata = MetaData() -event_log = Table( +event_log: Table = Table( "event_log", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -37,7 +38,7 @@ event_log = Table( class BaseData: - def __init__(self, cfg: CoreConfig, conn: Connection) -> None: + def __init__(self, cfg: CoreConfig, conn: AsyncSession) -> None: self.config = cfg self.conn = conn self.logger = logging.getLogger("database") @@ -47,7 +48,7 @@ class BaseData: try: self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}") - res = self.conn.execute(text(sql), opts) + res = await self.conn.execute(text(sql), opts) except SQLAlchemyError as e: self.logger.error(f"SQLAlchemy error {e}") @@ -59,7 +60,7 @@ class BaseData: except Exception: try: - res = self.conn.execute(sql, opts) + res = await self.conn.execute(sql, opts) except SQLAlchemyError as e: self.logger.error(f"SQLAlchemy error {e}") @@ -83,7 +84,7 @@ class BaseData: async def log_event( self, system: str, type: str, severity: int, message: str, details: Dict = {}, user: int = None, - arcade: int = None, machine: int = None, ip: str = None, game: str = None, version: str = None + arcade: int = None, machine: int = None, ip: Optional[str] = None, game: Optional[str] = None, version: Optional[str] = None ) -> Optional[int]: sql = event_log.insert().values( system=system, diff --git a/core/data/schema/card.py b/core/data/schema/card.py index 1865539..254b19e 100644 --- a/core/data/schema/card.py +++ b/core/data/schema/card.py @@ -1,13 +1,14 @@ from typing import Dict, List, Optional -from sqlalchemy import Table, Column, UniqueConstraint -from sqlalchemy.types import Integer, String, Boolean, TIMESTAMP, BIGINT, VARCHAR -from sqlalchemy.sql.schema import ForeignKey -from sqlalchemy.sql import func + +from sqlalchemy import Column, Table, UniqueConstraint from sqlalchemy.engine import Row +from sqlalchemy.sql import func +from sqlalchemy.sql.schema import ForeignKey +from sqlalchemy.types import BIGINT, TIMESTAMP, VARCHAR, Boolean, Integer, String from core.data.schema.base import BaseData, metadata -aime_card = Table( +aime_card: Table = Table( "aime_card", metadata, Column("id", Integer, primary_key=True, nullable=False), diff --git a/core/data/schema/user.py b/core/data/schema/user.py index 8c3695c..8686f08 100644 --- a/core/data/schema/user.py +++ b/core/data/schema/user.py @@ -1,15 +1,15 @@ -from typing import Optional, List -from sqlalchemy import Table, Column -from sqlalchemy.types import Integer, String, TIMESTAMP -from sqlalchemy.sql import func -from sqlalchemy.dialects.mysql import insert -from sqlalchemy.sql import func, select -from sqlalchemy.engine import Row +from typing import List, Optional + import bcrypt +from sqlalchemy import Column, Table +from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import Row +from sqlalchemy.sql import func, select +from sqlalchemy.types import TIMESTAMP, Integer, String from core.data.schema.base import BaseData, metadata -aime_user = Table( +aime_user: Table = Table( "aime_user", metadata, Column("id", Integer, nullable=False, primary_key=True, autoincrement=True), @@ -26,10 +26,10 @@ aime_user = Table( class UserData(BaseData): async def create_user( self, - id: int = None, - username: str = None, - email: str = None, - password: str = None, + id: Optional[int] = None, + username: Optional[str] = None, + email: Optional[str] = None, + password: Optional[str] = None, permission: int = 1, ) -> Optional[int]: if id is None: diff --git a/core/utils.py b/core/utils.py index 24c174c..af96451 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,18 +1,47 @@ -from typing import Dict, Any, Optional -from types import ModuleType -from starlette.requests import Request -import logging import importlib -from os import walk -import jwt +import logging from base64 import b64decode from datetime import datetime, timezone +from os import walk +from types import ModuleType +from typing import Any, Dict, Optional + +import jwt +from starlette.requests import Request from .config import CoreConfig + +class _MissingSentinel: + __slots__: tuple[str, ...] = () + + def __eq__(self, other) -> bool: + return False + + def __bool__(self) -> bool: + return False + + def __hash__(self) -> int: + return 0 + + def __repr__(self): + return "..." + + +MISSING: Any = _MissingSentinel() +"""This is different from `None` in that its type is `Any`, and so it can be used +as a placeholder for values that are *definitely* going to be initialized, +so they don't have to be typed as `T | None`, which makes type checkers +angry when an attribute is accessed. + +This can also be used for when `None` has actual meaning as a value, and so a +separate value is needed to mean "unset".""" + + class Utils: real_title_port = None real_title_port_ssl = None + @classmethod def get_all_titles(cls) -> Dict[str, ModuleType]: ret: Dict[str, Any] = {} @@ -36,27 +65,56 @@ class Utils: def get_ip_addr(cls, req: Request) -> str: ip = req.headers.get("x-forwarded-for", req.client.host) return ip.split(", ")[0] - + @classmethod def get_title_port(cls, cfg: CoreConfig): - if cls.real_title_port is not None: return cls.real_title_port + if cls.real_title_port is not None: + return cls.real_title_port + + cls.real_title_port = ( + cfg.server.proxy_port + if cfg.server.is_using_proxy and cfg.server.proxy_port + else cfg.server.port + ) - cls.real_title_port = cfg.server.proxy_port if cfg.server.is_using_proxy and cfg.server.proxy_port else cfg.server.port - return cls.real_title_port - + @classmethod def get_title_port_ssl(cls, cfg: CoreConfig): - if cls.real_title_port_ssl is not None: return cls.real_title_port_ssl + if cls.real_title_port_ssl is not None: + return cls.real_title_port_ssl + + cls.real_title_port_ssl = ( + cfg.server.proxy_port_ssl + if cfg.server.is_using_proxy and cfg.server.proxy_port_ssl + else 443 + ) - cls.real_title_port_ssl = cfg.server.proxy_port_ssl if cfg.server.is_using_proxy and cfg.server.proxy_port_ssl else 443 - return cls.real_title_port_ssl -def create_sega_auth_key(aime_id: int, game: str, place_id: int, keychip_id: str, b64_secret: str, exp_seconds: int = 86400, err_logger: str = 'aimedb') -> Optional[str]: + +def create_sega_auth_key( + aime_id: int, + game: str, + place_id: int, + keychip_id: str, + b64_secret: str, + exp_seconds: int = 86400, + err_logger: str = "aimedb", +) -> Optional[str]: logger = logging.getLogger(err_logger) try: - return jwt.encode({ "aime_id": aime_id, "game": game, "place_id": place_id, "keychip_id": keychip_id, "exp": int(datetime.now(tz=timezone.utc).timestamp()) + exp_seconds }, b64decode(b64_secret), algorithm="HS256") + return jwt.encode( + { + "aime_id": aime_id, + "game": game, + "place_id": place_id, + "keychip_id": keychip_id, + "exp": int(datetime.now(tz=timezone.utc).timestamp()) + exp_seconds, + }, + b64decode(b64_secret), + algorithm="HS256", + ) except jwt.InvalidKeyError: logger.error("Failed to encode Sega Auth Key because the secret is invalid!") return None @@ -64,10 +122,19 @@ def create_sega_auth_key(aime_id: int, game: str, place_id: int, keychip_id: str logger.error(f"Unknown exception occoured when encoding Sega Auth Key! {e}") return None -def decode_sega_auth_key(token: str, b64_secret: str, err_logger: str = 'aimedb') -> Optional[Dict]: + +def decode_sega_auth_key( + token: str, b64_secret: str, err_logger: str = "aimedb" +) -> Optional[Dict]: logger = logging.getLogger(err_logger) try: - return jwt.decode(token, "secret", b64decode(b64_secret), algorithms=["HS256"], options={"verify_signature": True}) + return jwt.decode( + token, + "secret", + b64decode(b64_secret), + algorithms=["HS256"], + options={"verify_signature": True}, + ) except jwt.ExpiredSignatureError: logger.error("Sega Auth Key failed to validate due to an expired signature!") return None @@ -83,4 +150,3 @@ def decode_sega_auth_key(token: str, b64_secret: str, err_logger: str = 'aimedb' except Exception as e: logger.error(f"Unknown exception occoured when decoding Sega Auth Key! {e}") return None - \ No newline at end of file diff --git a/dbutils.py b/dbutils.py index 9314f8e..9080afc 100644 --- a/dbutils.py +++ b/dbutils.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import argparse -import logging -from os import mkdir, path, access, W_OK, environ -import yaml import asyncio +import logging +from os import W_OK, access, environ, mkdir, path + +import yaml -from core.data import Data from core.config import CoreConfig +from core.data import Data if __name__ == "__main__": parser = argparse.ArgumentParser(description="Database utilities") @@ -46,7 +47,7 @@ if __name__ == "__main__": loop = asyncio.get_event_loop() if args.action == "create": - data.create_database() + loop.run_until_complete(data.create_database()) elif args.action == "upgrade": data.schema_upgrade(args.version) diff --git a/read.py b/read.py index 8a0ae72..c6950a2 100644 --- a/read.py +++ b/read.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import argparse -import re -import os -import yaml -from os import path -import logging -import coloredlogs import asyncio - +import logging +import os +import re from logging.handlers import TimedRotatingFileHandler +from os import path from typing import List, Optional +import coloredlogs +import yaml + from core import CoreConfig, Utils @@ -44,7 +44,7 @@ class BaseReader: pass -if __name__ == "__main__": +async def main(): parser = argparse.ArgumentParser(description="Import Game Information") parser.add_argument( "--game", @@ -140,8 +140,12 @@ if __name__ == "__main__": for dir, mod in titles.items(): if args.game in mod.game_codes: handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra) - loop = asyncio.get_event_loop() - loop.run_until_complete(handler.read()) + + await handler.read() logger.info("Done") + + +if __name__ == "__main__": + asyncio.run(main()) From 4c33f4282a068b3bdbab1f6a51bfd45d5ae69521 Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 14 Nov 2024 12:38:00 +0700 Subject: [PATCH 058/130] oops forgot a dependency on aiomysql --- requirements.txt | Bin 265 -> 263 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index fe5b4efd44150f88955d20cd3f7d4df314a8990c..4a10c44288d0a228f9ce2b4b24f7933841527445 100644 GIT binary patch delta 20 bcmeBVYG<0D#gUkqpIcd6m^0Dz3L_T)M*Rls delta 21 ccmZo?>SUUr#hqJOT$q!blbM=VGSTP?08jr2CIA2c From 789d50c406560068d4677e82d9c017bceb7183ae Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 14 Nov 2024 13:10:14 +0700 Subject: [PATCH 059/130] use AsyncSession directly see the warnings in https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html#using-asyncio-scoped-session --- core/data/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/data/database.py b/core/data/database.py index 16bd67b..0095a20 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -1,4 +1,3 @@ -import asyncio import logging import os import secrets @@ -15,10 +14,8 @@ import pymysql.err from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, - async_scoped_session, create_async_engine, ) -from sqlalchemy.orm import sessionmaker from core.config import CoreConfig from core.data.schema import ArcadeData, BaseData, CardData, UserData, metadata @@ -47,8 +44,7 @@ class Data: self.__engine = Data.engine if Data.session is MISSING: - s = sessionmaker(Data.engine, expire_on_commit=False, class_=AsyncSession) - Data.session = async_scoped_session(s, asyncio.current_task) + Data.session = AsyncSession(Data.engine, expire_on_commit=False) if Data.user is MISSING: Data.user = UserData(self.config, self.session) From cb009f6e2356afc9726a0c000a9d5a878cd61662 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 14 Nov 2024 12:39:21 -0500 Subject: [PATCH 060/130] wacca: tiny cleanup --- titles/wacca/index.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/titles/wacca/index.py b/titles/wacca/index.py index 97a3fd4..83264eb 100644 --- a/titles/wacca/index.py +++ b/titles/wacca/index.py @@ -98,9 +98,7 @@ class WaccaServlet(BaseServlet): async def render_POST(self, request: Request) -> bytes: def end(resp: Dict) -> bytes: hash = md5(json.dumps(resp, ensure_ascii=False).encode()).digest() - j_Resp = Response(json.dumps(resp, ensure_ascii=False)) - j_Resp.raw_headers.append((b"X-Wacca-Hash", hash.hex().encode())) - return j_Resp + return Response(content=json.dumps(resp, ensure_ascii=False), headers={"X-Wacca-Hash": hash.hex()}) api = request.path_params.get('api', '') branch = request.path_params.get('branch', '') From 58a5177a30dc8472c1f9cd60d6bb238b3c03c49e Mon Sep 17 00:00:00 2001 From: beerpsi Date: Sat, 16 Nov 2024 19:10:29 +0000 Subject: [PATCH 061/130] use SQL's limit/offset pagination for nextIndex/maxCount requests (#185) Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need. Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready. Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client. ### How this works Say we have a `GetUserCharacterApi` request: ```json { "userId": 10000, "maxCount": 700, "nextIndex": 0 } ``` Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query: ```python select(character) .where(character.c.user == user_id) .order_by(character.c.id.asc()) .limit(max_count + 1) .offset(next_index) ``` The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned: ```python rows = ... if len(rows) > max_count: # return only max_count rows next_index += max_count else: # return everything left next_index = -1 ``` This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](https://gitea.tendokyu.moe/Hay1tsme/artemis/src/commit/2274b42358d9ef449ca541a46ce654b846ce7f7c/titles/chuni/base.py#L13).) Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185 Co-authored-by: beerpsi Co-committed-by: beerpsi --- core/config.py | 107 ++++++++- core/data/alembic/env.py | 16 +- core/data/database.py | 17 +- dbutils.py | 18 +- titles/chuni/base.py | 291 ++++++++++++----------- titles/chuni/const.py | 54 ++++- titles/chuni/new.py | 36 +-- titles/chuni/schema/item.py | 89 +++++-- titles/chuni/schema/score.py | 82 +++++-- titles/mai2/base.py | 276 +++++++++++++--------- titles/mai2/dx.py | 427 ++++++++++++++++++++-------------- titles/mai2/schema/item.py | 193 +++++++++++---- titles/mai2/schema/profile.py | 48 +++- titles/mai2/schema/score.py | 75 ++++-- titles/ongeki/base.py | 189 +++++++++------ titles/ongeki/bright.py | 86 +++---- titles/ongeki/schema/item.py | 70 ++++-- titles/ongeki/schema/score.py | 49 +++- 18 files changed, 1410 insertions(+), 713 deletions(-) diff --git a/core/config.py b/core/config.py index dda43aa..e05323b 100644 --- a/core/config.py +++ b/core/config.py @@ -1,5 +1,9 @@ -import logging, os -from typing import Any +import logging +import os +import ssl +from typing import Any, Union + +from typing_extensions import Optional class ServerConfig: def __init__(self, parent_config: "CoreConfig") -> None: @@ -175,12 +179,60 @@ class DatabaseConfig: return CoreConfig.get_config_field( self.__config, "core", "database", "protocol", default="mysql" ) - + @property - def ssl_enabled(self) -> str: + def ssl_enabled(self) -> bool: return CoreConfig.get_config_field( self.__config, "core", "database", "ssl_enabled", default=False ) + + @property + def ssl_cafile(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_cafile", default=None + ) + + @property + def ssl_capath(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_capath", default=None + ) + + @property + def ssl_cert(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_cert", default=None + ) + + @property + def ssl_key(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_key", default=None + ) + + @property + def ssl_key_password(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_key_password", default=None + ) + + @property + def ssl_verify_identity(self) -> bool: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_verify_identity", default=True + ) + + @property + def ssl_verify_cert(self) -> Optional[Union[str, bool]]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_verify_cert", default=None + ) + + @property + def ssl_ciphers(self) -> Optional[str]: + return CoreConfig.get_config_field( + self.__config, "core", "database", "ssl_ciphers", default=None + ) @property def sha2_password(self) -> bool: @@ -208,6 +260,53 @@ class DatabaseConfig: self.__config, "core", "database", "memcached_host", default="localhost" ) + def create_ssl_context_if_enabled(self): + if not self.ssl_enabled: + return + + no_ca = ( + self.ssl_cafile is None + and self.ssl_capath is None + ) + + ctx = ssl.create_default_context( + cafile=self.ssl_cafile, + capath=self.ssl_capath, + ) + ctx.check_hostname = not no_ca and self.ssl_verify_identity + + if self.ssl_verify_cert is None: + ctx.verify_mode = ssl.CERT_NONE if no_ca else ssl.CERT_REQUIRED + elif isinstance(self.ssl_verify_cert, bool): + ctx.verify_mode = ( + ssl.CERT_REQUIRED + if self.ssl_verify_cert + else ssl.CERT_NONE + ) + elif isinstance(self.ssl_verify_cert, str): + value = self.ssl_verify_cert.lower() + + if value in ("none", "0", "false", "no"): + ctx.verify_mode = ssl.CERT_NONE + elif value == "optional": + ctx.verify_mode = ssl.CERT_OPTIONAL + elif value in ("required", "1", "true", "yes"): + ctx.verify_mode = ssl.CERT_REQUIRED + else: + ctx.verify_mode = ssl.CERT_NONE if no_ca else ssl.CERT_REQUIRED + + if self.ssl_cert: + ctx.load_cert_chain( + self.ssl_cert, + self.ssl_key, + self.ssl_key_password, + ) + + if self.ssl_ciphers: + ctx.set_ciphers(self.ssl_ciphers) + + return ctx + class FrontendConfig: def __init__(self, parent_config: "CoreConfig") -> None: self.__config = parent_config diff --git a/core/data/alembic/env.py b/core/data/alembic/env.py index f2a8182..b175ee6 100644 --- a/core/data/alembic/env.py +++ b/core/data/alembic/env.py @@ -1,14 +1,18 @@ from __future__ import with_statement import asyncio +import os +from pathlib import Path import threading from logging.config import fileConfig +import yaml from alembic import context from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config +from core.config import CoreConfig from core.data.schema.base import metadata # this is the Alembic Config object, which provides @@ -74,8 +78,18 @@ async def run_async_migrations() -> None: for override in overrides: ini_section[override] = overrides[override] + core_config = CoreConfig() + + with (Path("../../..") / os.environ["ARTEMIS_CFG_DIR"] / "core.yaml").open(encoding="utf-8") as f: + core_config.update(yaml.safe_load(f)) + connectable = async_engine_from_config( - ini_section, prefix="sqlalchemy.", poolclass=pool.NullPool + ini_section, + poolclass=pool.NullPool, + connect_args={ + "charset": "utf8mb4", + "ssl": core_config.database.create_ssl_context_if_enabled(), + } ) async with connectable.connect() as connection: diff --git a/core/data/database.py b/core/data/database.py index 0095a20..170665e 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -1,11 +1,12 @@ import logging import os import secrets +import ssl import string import warnings from hashlib import sha256 from logging.handlers import TimedRotatingFileHandler -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional import alembic.config import bcrypt @@ -35,12 +36,20 @@ class Data: if self.config.database.sha2_password: passwd = sha256(self.config.database.password.encode()).digest() - self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" + self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}" else: - self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}?charset=utf8mb4&ssl={str(self.config.database.ssl_enabled).lower()}" + self.__url = f"{self.config.database.protocol}+aiomysql://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}:{self.config.database.port}/{self.config.database.name}" if Data.engine is MISSING: - Data.engine = create_async_engine(self.__url, pool_recycle=3600, isolation_level="AUTOCOMMIT") + Data.engine = create_async_engine( + self.__url, + pool_recycle=3600, + isolation_level="AUTOCOMMIT", + connect_args={ + "charset": "utf8mb4", + "ssl": self.config.database.create_ssl_context_if_enabled(), + }, + ) self.__engine = Data.engine if Data.session is MISSING: diff --git a/dbutils.py b/dbutils.py index 9080afc..154df0a 100644 --- a/dbutils.py +++ b/dbutils.py @@ -9,7 +9,7 @@ import yaml from core.config import CoreConfig from core.data import Data -if __name__ == "__main__": +async def main(): parser = argparse.ArgumentParser(description="Database utilities") parser.add_argument( "--config", "-c", type=str, help="Config folder to use", default="config" @@ -44,10 +44,8 @@ if __name__ == "__main__": data = Data(cfg) - loop = asyncio.get_event_loop() - if args.action == "create": - loop.run_until_complete(data.create_database()) + await data.create_database() elif args.action == "upgrade": data.schema_upgrade(args.version) @@ -59,16 +57,20 @@ if __name__ == "__main__": data.schema_downgrade(args.version) elif args.action == "create-owner": - loop.run_until_complete(data.create_owner(args.email, args.access_code)) + await data.create_owner(args.email, args.access_code) elif args.action == "migrate": - loop.run_until_complete(data.migrate()) + await data.migrate() elif args.action == "create-revision": - loop.run_until_complete(data.create_revision(args.message)) + await data.create_revision(args.message) elif args.action == "create-autorevision": - loop.run_until_complete(data.create_revision_auto(args.message)) + await data.create_revision_auto(args.message) else: logging.getLogger("database").info(f"Unknown action {args.action}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/titles/chuni/base.py b/titles/chuni/base.py index 2410ef0..37bcbb3 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -1,16 +1,16 @@ -import logging +import itertools import json +import logging from datetime import datetime, timedelta -from time import strftime +from typing import Any, Dict, List import pytz -from typing import Dict, Any, List from core.config import CoreConfig +from titles.chuni.config import ChuniConfig from titles.chuni.const import ChuniConstants, ItemKind from titles.chuni.database import ChuniData -from titles.chuni.config import ChuniConfig -SCORE_BUFFER = {} + class ChuniBase: def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None: @@ -277,35 +277,39 @@ class ChuniBase: } async def handle_get_user_character_api_request(self, data: Dict) -> Dict: - characters = await self.data.item.get_characters(data["userId"]) - if characters is None: + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + + # add one to the limit so we know if there's a next page of items + rows = await self.data.item.get_characters( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None or len(rows) == 0: return { - "userId": data["userId"], + "userId": user_id, "length": 0, "nextIndex": -1, "userCharacterList": [], } character_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(characters)): - tmp = characters[x]._asdict() - tmp.pop("user") + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("id") + tmp.pop("user") + character_list.append(tmp) - if len(character_list) >= max_ct: - break - - if len(characters) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = -1 return { - "userId": data["userId"], + "userId": user_id, "length": len(character_list), "nextIndex": next_idx, "userCharacterList": character_list, @@ -335,29 +339,31 @@ class ChuniBase: } async def handle_get_user_course_api_request(self, data: Dict) -> Dict: - user_course_list = await self.data.score.get_courses(data["userId"]) - if user_course_list is None: + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + + rows = await self.data.score.get_courses( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None or len(rows) == 0: return { - "userId": data["userId"], + "userId": user_id, "length": 0, "nextIndex": -1, "userCourseList": [], } course_list = [] - next_idx = int(data.get("nextIndex", 0)) - max_ct = int(data.get("maxCount", 300)) - for x in range(next_idx, len(user_course_list)): - tmp = user_course_list[x]._asdict() + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("user") tmp.pop("id") course_list.append(tmp) - if len(user_course_list) >= max_ct: - break - - if len(user_course_list) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = -1 @@ -425,75 +431,94 @@ class ChuniBase: } async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict: - rival_id = data["rivalId"] - next_index = int(data["nextIndex"]) - max_count = int(data["maxCount"]) - user_rival_music_list = [] + user_id = int(data["userId"]) + rival_id = int(data["rivalId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + rival_levels = [int(x["level"]) for x in data["userRivalMusicLevelList"]] # Fetch all the rival music entries for the user - all_entries = await self.data.score.get_rival_music(rival_id) + rows = await self.data.score.get_scores( + rival_id, + levels=rival_levels, + limit=max_ct + 1, + offset=next_idx, + ) - # Process the entries based on max_count and nextIndex - for music in all_entries: - music_id = music["musicId"] - level = music["level"] - score = music["scoreMax"] - rank = music["scoreRank"] + if rows is None or len(rows) == 0: + return { + "userId": user_id, + "rivalId": rival_id, + "nextIndex": -1, + "userRivalMusicList": [], + } - # Create a music entry for the current music_id if it's unique - music_entry = next((entry for entry in user_rival_music_list if entry["musicId"] == music_id), None) - if music_entry is None: - music_entry = { - "musicId": music_id, - "length": 0, - "userRivalMusicDetailList": [] - } - user_rival_music_list.append(music_entry) + music_details = [x._asdict() for x in rows] + returned_music_details_count = 0 + music_list = [] - # Create a level entry for the current level if it's unique or has a higher score - level_entry = next((entry for entry in music_entry["userRivalMusicDetailList"] if entry["level"] == level), None) - if level_entry is None: - level_entry = { - "level": level, - "scoreMax": score, - "scoreRank": rank - } - music_entry["userRivalMusicDetailList"].append(level_entry) - elif score > level_entry["scoreMax"]: - level_entry["scoreMax"] = score - level_entry["scoreRank"] = rank + # note that itertools.groupby will only work on sorted keys, which is already sorted by + # the query in get_scores + for music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]): + details: list[dict[Any, Any]] = [ + {"level": d["level"], "scoreMax": d["scoreMax"]} + for d in details_iter + ] - # Calculate the length for each "musicId" by counting the unique levels - for music_entry in user_rival_music_list: - music_entry["length"] = len(music_entry["userRivalMusicDetailList"]) + music_list.append({"musicId": music_id, "length": len(details), "userMusicDetailList": details}) + returned_music_details_count += len(details) - # Prepare the result dictionary with user rival music data - result = { - "userId": data["userId"], - "rivalId": data["rivalId"], - "nextIndex": str(next_index + len(user_rival_music_list[next_index: next_index + max_count]) if max_count <= len(user_rival_music_list[next_index: next_index + max_count]) else -1), - "userRivalMusicList": user_rival_music_list[next_index: next_index + max_count] + if len(music_list) >= max_ct: + break + + # if we returned fewer PBs than we originally asked for from the database, that means + # we queried for the PBs of max_ct + 1 songs. + if returned_music_details_count < len(rows): + next_idx += max_ct + else: + next_idx = -1 + + return { + "userId": user_id, + "rivalId": rival_id, + "length": len(music_list), + "nextIndex": next_idx, + "userRivalMusicList": music_list, } - return result - async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict: + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + kind = int(data["kind"]) + is_all_favorite_item = str(data["isAllFavoriteItem"]) == "true" + user_fav_item_list = [] # still needs to be implemented on WebUI # 1: Music, 2: User, 3: Character - fav_list = await self.data.item.get_all_favorites( - data["userId"], self.version, fav_kind=int(data["kind"]) + rows = await self.data.item.get_all_favorites( + user_id, + self.version, + fav_kind=kind, + limit=max_ct + 1, + offset=next_idx, ) - if fav_list is not None: - for fav in fav_list: + + if rows is not None: + for fav in rows[:max_ct]: user_fav_item_list.append({"id": fav["favId"]}) + if rows is None or len(rows) <= max_ct: + next_idx = -1 + else: + next_idx += max_ct + return { - "userId": data["userId"], + "userId": user_id, "length": len(user_fav_item_list), - "kind": data["kind"], - "nextIndex": -1, + "kind": kind, + "nextIndex": next_idx, "userFavoriteItemList": user_fav_item_list, } @@ -505,36 +530,39 @@ class ChuniBase: return {"userId": data["userId"], "length": 0, "userFavoriteMusicList": []} async def handle_get_user_item_api_request(self, data: Dict) -> Dict: - kind = int(int(data["nextIndex"]) / 10000000000) - next_idx = int(int(data["nextIndex"]) % 10000000000) - user_item_list = await self.data.item.get_items(data["userId"], kind) + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) - if user_item_list is None or len(user_item_list) == 0: + kind = next_idx // 10000000000 + next_idx = next_idx % 10000000000 + rows = await self.data.item.get_items( + user_id, kind, limit=max_ct + 1, offset=next_idx + ) + + if rows is None or len(rows) == 0: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": -1, "itemKind": kind, "userItemList": [], } items: List[Dict[str, Any]] = [] - for i in range(next_idx, len(user_item_list)): - tmp = user_item_list[i]._asdict() + + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("user") tmp.pop("id") items.append(tmp) - if len(items) >= int(data["maxCount"]): - break - xout = kind * 10000000000 + next_idx + len(items) - - if len(items) < int(data["maxCount"]): - next_idx = 0 + if len(rows) > max_ct: + next_idx = kind * 10000000000 + next_idx + max_ct else: - next_idx = xout + next_idx = -1 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "itemKind": kind, "length": len(items), @@ -586,62 +614,55 @@ class ChuniBase: } async def handle_get_user_music_api_request(self, data: Dict) -> Dict: - music_detail = await self.data.score.get_scores(data["userId"]) - if music_detail is None: + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + + rows = await self.data.score.get_scores( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None or len(rows) == 0: return { - "userId": data["userId"], + "userId": user_id, "length": 0, "nextIndex": -1, "userMusicList": [], # 240 } - song_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) + music_details = [x._asdict() for x in rows] + returned_music_details_count = 0 + music_list = [] - for x in range(next_idx, len(music_detail)): - found = False - tmp = music_detail[x]._asdict() - tmp.pop("user") - tmp.pop("id") + # note that itertools.groupby will only work on sorted keys, which is already sorted by + # the query in get_scores + for _music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]): + details: list[dict[Any, Any]] = [] - for song in song_list: - score_buf = SCORE_BUFFER.get(str(data["userId"])) or [] - if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]: - found = True - song["userMusicDetailList"].append(tmp) - song["length"] = len(song["userMusicDetailList"]) - score_buf.append(tmp["musicId"]) - SCORE_BUFFER[str(data["userId"])] = score_buf + for d in details_iter: + d.pop("id") + d.pop("user") - score_buf = SCORE_BUFFER.get(str(data["userId"])) or [] - if not found and tmp["musicId"] not in score_buf: - song_list.append({"length": 1, "userMusicDetailList": [tmp]}) - score_buf.append(tmp["musicId"]) - SCORE_BUFFER[str(data["userId"])] = score_buf + details.append(d) - if len(song_list) >= max_ct: + music_list.append({"length": len(details), "userMusicDetailList": details}) + returned_music_details_count += len(details) + + if len(music_list) >= max_ct: break - - for songIdx in range(len(song_list)): - for recordIdx in range(x+1, len(music_detail)): - if song_list[songIdx]["userMusicDetailList"][0]["musicId"] == music_detail[recordIdx]["musicId"]: - music = music_detail[recordIdx]._asdict() - music.pop("user") - music.pop("id") - song_list[songIdx]["userMusicDetailList"].append(music) - song_list[songIdx]["length"] += 1 - - if len(song_list) >= max_ct: - next_idx += len(song_list) + + # if we returned fewer PBs than we originally asked for from the database, that means + # we queried for the PBs of max_ct + 1 songs. + if returned_music_details_count < len(rows): + next_idx += max_ct else: next_idx = -1 - SCORE_BUFFER[str(data["userId"])] = [] + return { - "userId": data["userId"], - "length": len(song_list), + "userId": user_id, + "length": len(music_list), "nextIndex": next_idx, - "userMusicList": song_list, # 240 + "userMusicList": music_list, } async def handle_get_user_option_api_request(self, data: Dict) -> Dict: diff --git a/titles/chuni/const.py b/titles/chuni/const.py index 68d7056..45fd498 100644 --- a/titles/chuni/const.py +++ b/titles/chuni/const.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import Enum, IntEnum class ChuniConstants: @@ -81,12 +81,31 @@ class ChuniConstants: return cls.VERSION_NAMES[ver] -class MapAreaConditionType(Enum): - UNLOCKED = 0 +class MapAreaConditionType(IntEnum): + """Condition types for the GetGameMapAreaConditionApi endpoint. Incomplete. + + For the MAP_CLEARED/MAP_AREA_CLEARED/TROPHY_OBTAINED conditions, the conditionId + is the map/map area/trophy. + + For the RANK_*/ALL_JUSTICE conditions, the conditionId is songId * 100 + difficultyId. + For example, Halcyon [ULTIMA] would be 173 * 100 + 4 = 17304. + """ + + ALWAYS_UNLOCKED = 0 + MAP_CLEARED = 1 MAP_AREA_CLEARED = 2 + TROPHY_OBTAINED = 3 + RANK_SSS = 19 + RANK_SSP = 20 + RANK_SS = 21 + RANK_SP = 22 + RANK_S = 23 + + ALL_JUSTICE = 28 + class MapAreaConditionLogicalOperator(Enum): AND = 1 @@ -102,11 +121,36 @@ class AvatarCategory(Enum): FRONT = 6 BACK = 7 -class ItemKind(Enum): +class ItemKind(IntEnum): NAMEPLATE = 1 + + FRAME = 2 + """ + "Frame" is the background for the gauge/score/max combo display + shown during gameplay. This item cannot be equipped (as of LUMINOUS) + and is hardcoded to the current game's version. + """ + TROPHY = 3 + SKILL = 4 + TICKET = 5 + """A statue is also a ticket.""" + PRESENT = 6 + MUSIC_UNLOCK = 7 MAP_ICON = 8 SYSTEM_VOICE = 9 - AVATAR_ACCESSORY = 11 \ No newline at end of file + SYMBOL_CHAT = 10 + AVATAR_ACCESSORY = 11 + + ULTIMA_UNLOCK = 12 + """This only applies to ULTIMA difficulties that are *not* unlocked by + SS-ing EXPERT+MASTER. + """ + + +class FavoriteItemKind(IntEnum): + MUSIC = 1 + RIVAL = 2 + CHARACTER = 3 diff --git a/titles/chuni/new.py b/titles/chuni/new.py index 3d3fb98..15d2b6c 100644 --- a/titles/chuni/new.py +++ b/titles/chuni/new.py @@ -4,12 +4,14 @@ from random import randint from typing import Dict import pytz + from core.config import CoreConfig from core.utils import Utils -from titles.chuni.const import ChuniConstants -from titles.chuni.database import ChuniData from titles.chuni.base import ChuniBase from titles.chuni.config import ChuniConfig +from titles.chuni.const import ChuniConstants +from titles.chuni.database import ChuniData + class ChuniNew(ChuniBase): ITEM_TYPE = {"character": 20, "story": 21, "card": 22} @@ -285,35 +287,37 @@ class ChuniNew(ChuniBase): } async def handle_get_user_printed_card_api_request(self, data: Dict) -> Dict: - user_print_list = await self.data.item.get_user_print_states( - data["userId"], has_completed=True + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) + + rows = await self.data.item.get_user_print_states( + user_id, + has_completed=True, + limit=max_ct + 1, + offset=next_idx, ) - if user_print_list is None: + if rows is None or len(rows) == 0: return { - "userId": data["userId"], + "userId": user_id, "length": 0, "nextIndex": -1, "userPrintedCardList": [], } print_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(user_print_list)): - tmp = user_print_list[x]._asdict() + for row in rows[:max_ct]: + tmp = row._asdict() print_list.append(tmp["cardId"]) - if len(print_list) >= max_ct: - break - - if len(print_list) >= max_ct: - next_idx = next_idx + max_ct + if len(rows) > max_ct: + next_idx += max_ct else: next_idx = -1 return { - "userId": data["userId"], + "userId": user_id, "length": len(print_list), "nextIndex": next_idx, "userPrintedCardList": print_list, diff --git a/titles/chuni/schema/item.py b/titles/chuni/schema/item.py index 92910da..93dcf86 100644 --- a/titles/chuni/schema/item.py +++ b/titles/chuni/schema/item.py @@ -1,22 +1,22 @@ from typing import Dict, List, Optional + from sqlalchemy import ( - Table, Column, - UniqueConstraint, PrimaryKeyConstraint, + Table, + UniqueConstraint, and_, delete, ) -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON -from sqlalchemy.engine.base import Connection -from sqlalchemy.schema import ForeignKey -from sqlalchemy.sql import func, select from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row +from sqlalchemy.schema import ForeignKey +from sqlalchemy.sql import func, select +from sqlalchemy.types import JSON, TIMESTAMP, Boolean, Integer, String from core.data.schema import BaseData, metadata -character = Table( +character: Table = Table( "chuni_item_character", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -40,7 +40,7 @@ character = Table( mysql_charset="utf8mb4", ) -item = Table( +item: Table = Table( "chuni_item_item", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -141,7 +141,7 @@ gacha = Table( mysql_charset="utf8mb4", ) -print_state = Table( +print_state: Table = Table( "chuni_item_print_state", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -210,7 +210,7 @@ login_bonus = Table( mysql_charset="utf8mb4", ) -favorite = Table( +favorite: Table = Table( "chuni_item_favorite", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -379,9 +379,14 @@ class ChuniItemData(BaseData): return True if len(result.all()) else False async def get_all_favorites( - self, user_id: int, version: int, fav_kind: int = 1 + self, + user_id: int, + version: int, + fav_kind: int = 1, + limit: Optional[int] = None, + offset: Optional[int] = None, ) -> Optional[List[Row]]: - sql = favorite.select( + sql = select(favorite).where( and_( favorite.c.version == version, favorite.c.user == user_id, @@ -389,6 +394,13 @@ class ChuniItemData(BaseData): ) ) + if limit is not None or offset is not None: + sql = sql.order_by(favorite.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None @@ -488,9 +500,18 @@ class ChuniItemData(BaseData): return None return result.fetchone() - async def get_characters(self, user_id: int) -> Optional[List[Row]]: + async def get_characters( + self, user_id: int, limit: Optional[int] = None, offset: Optional[int] = None + ) -> Optional[List[Row]]: sql = select(character).where(character.c.user == user_id) + if limit is not None or offset is not None: + sql = sql.order_by(character.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None @@ -509,13 +530,26 @@ class ChuniItemData(BaseData): return None return result.lastrowid - async def get_items(self, user_id: int, kind: int = None) -> Optional[List[Row]]: - if kind is None: - sql = select(item).where(item.c.user == user_id) - else: - sql = select(item).where( - and_(item.c.user == user_id, item.c.itemKind == kind) - ) + async def get_items( + self, + user_id: int, + kind: Optional[int] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + cond = item.c.user == user_id + + if kind is not None: + cond &= item.c.itemKind == kind + + sql = select(item).where(cond) + + if limit is not None or offset is not None: + sql = sql.order_by(item.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -609,15 +643,26 @@ class ChuniItemData(BaseData): return result.lastrowid async def get_user_print_states( - self, aime_id: int, has_completed: bool = False + self, + aime_id: int, + has_completed: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, ) -> Optional[List[Row]]: - sql = print_state.select( + sql = select(print_state).where( and_( print_state.c.user == aime_id, print_state.c.hasCompleted == has_completed, ) ) + if limit is not None or offset is not None: + sql = sql.order_by(print_state.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None diff --git a/titles/chuni/schema/score.py b/titles/chuni/schema/score.py index 308afa8..ab6766a 100644 --- a/titles/chuni/schema/score.py +++ b/titles/chuni/schema/score.py @@ -1,16 +1,17 @@ from typing import Dict, List, Optional -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger -from sqlalchemy.engine.base import Connection -from sqlalchemy.schema import ForeignKey -from sqlalchemy.engine import Row -from sqlalchemy.sql import func, select + +from sqlalchemy import Column, Table, UniqueConstraint from sqlalchemy.dialects.mysql import insert -from sqlalchemy.sql.expression import exists +from sqlalchemy.engine import Row +from sqlalchemy.schema import ForeignKey +from sqlalchemy.sql import func, select +from sqlalchemy.types import Boolean, Integer, String + from core.data.schema import BaseData, metadata + from ..config import ChuniConfig -course = Table( +course: Table = Table( "chuni_score_course", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -41,7 +42,7 @@ course = Table( mysql_charset="utf8mb4", ) -best_score = Table( +best_score: Table = Table( "chuni_score_best", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -229,9 +230,21 @@ class ChuniRomVersion(): return -1 class ChuniScoreData(BaseData): - async def get_courses(self, aime_id: int) -> Optional[Row]: + async def get_courses( + self, + aime_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: sql = select(course).where(course.c.user == aime_id) + if limit is not None or offset is not None: + sql = sql.order_by(course.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None @@ -249,8 +262,45 @@ class ChuniScoreData(BaseData): return None return result.lastrowid - async def get_scores(self, aime_id: int) -> Optional[Row]: - sql = select(best_score).where(best_score.c.user == aime_id) + async def get_scores( + self, + aime_id: int, + levels: Optional[list[int]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + condition = best_score.c.user == aime_id + + if levels is not None: + condition &= best_score.c.level.in_(levels) + + if limit is None and offset is None: + sql = ( + select(best_score) + .where(condition) + .order_by(best_score.c.musicId.asc(), best_score.c.level.asc()) + ) + else: + subq = ( + select(best_score.c.musicId) + .distinct() + .where(condition) + .order_by(best_score.c.musicId) + ) + + if limit is not None: + subq = subq.limit(limit) + if offset is not None: + subq = subq.offset(offset) + + subq = subq.subquery() + + sql = ( + select(best_score) + .join(subq, best_score.c.musicId == subq.c.musicId) + .where(condition) + .order_by(best_score.c.musicId, best_score.c.level) + ) result = await self.execute(sql) if result is None: @@ -360,11 +410,3 @@ class ChuniScoreData(BaseData): rows = result.fetchall() return [dict(row) for row in rows] - - async def get_rival_music(self, rival_id: int) -> Optional[List[Dict]]: - sql = select(best_score).where(best_score.c.user == rival_id) - - result = await self.execute(sql) - if result is None: - return None - return result.fetchall() diff --git a/titles/mai2/base.py b/titles/mai2/base.py index 1c9d91c..add3ed2 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -1,16 +1,17 @@ -from datetime import datetime, timedelta -from typing import Any, Dict, List +import itertools import logging from base64 import b64decode -from os import path, stat, remove, mkdir, access, W_OK -from PIL import ImageFile -from random import randint +from datetime import datetime, timedelta +from os import W_OK, access, mkdir, path +from typing import Any, Dict, List import pytz + from core.config import CoreConfig from core.utils import Utils -from .const import Mai2Constants + from .config import Mai2Config +from .const import Mai2Constants from .database import Mai2Data @@ -444,23 +445,22 @@ class Mai2Base: return {"userId": data["userId"], "userOption": options_dict} async def handle_get_user_card_api_request(self, data: Dict) -> Dict: - user_cards = await self.data.item.get_cards(data["userId"]) - if user_cards is None: - return {"userId": data["userId"], "nextIndex": 0, "userCardList": []} + user_id = int(data["userId"]) + next_idx = int(data["nextIndex"]) + max_ct = int(data["maxCount"]) - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx + user_cards = await self.data.item.get_cards( + user_id, limit=max_ct + 1, offset=next_idx + ) - if len(user_cards[start_idx:]) > max_ct: - next_idx += max_ct - else: - next_idx = 0 + if user_cards is None or len(user_cards) == 0: + return {"userId": user_id, "nextIndex": 0, "userCardList": []} card_list = [] - for card in user_cards: + + for card in user_cards[:max_ct]: tmp = card._asdict() + tmp.pop("id") tmp.pop("user") tmp["startDate"] = datetime.strftime( @@ -469,12 +469,18 @@ class Mai2Base: tmp["endDate"] = datetime.strftime( tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT ) + card_list.append(tmp) + if len(user_cards) > max_ct: + next_idx += max_ct + else: + next_idx = 0 + return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, - "userCardList": card_list[start_idx:end_idx], + "userCardList": card_list, } async def handle_get_user_charge_api_request(self, data: Dict) -> Dict: @@ -536,28 +542,35 @@ class Mai2Base: return { "userId": data.get("userId", 0), "userBossData": boss_lst} async def handle_get_user_item_api_request(self, data: Dict) -> Dict: - kind = int(data["nextIndex"] / 10000000000) - next_idx = int(data["nextIndex"] % 10000000000) - user_item_list = await self.data.item.get_items(data["userId"], kind) + user_id: int = data["userId"] + kind: int = data["nextIndex"] // 10000000000 + next_idx: int = data["nextIndex"] % 10000000000 + max_ct: int = data["maxCount"] + rows = await self.data.item.get_items(user_id, kind, limit=max_ct, offset=next_idx) + + if rows is None or len(rows) == 0: + return { + "userId": user_id, + "nextIndex": 0, + "itemKind": kind, + "userItemList": [], + } items: List[Dict[str, Any]] = [] - for i in range(next_idx, len(user_item_list)): - tmp = user_item_list[i]._asdict() + + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("user") tmp.pop("id") items.append(tmp) - if len(items) >= int(data["maxCount"]): - break - xout = kind * 10000000000 + next_idx + len(items) - - if len(items) < int(data["maxCount"]): - next_idx = 0 + if len(rows) > max_ct: + next_idx = kind * 10000000000 + next_idx + max_ct else: - next_idx = xout + next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "itemKind": kind, "userItemList": items, @@ -675,77 +688,90 @@ class Mai2Base: return {"length": 0, "userPortraitList": []} async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict: - friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"]) - if friend_season_ranking is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_friend_season_ranking( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": 0, "userFriendSeasonRankingList": [], } friend_season_ranking_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(friend_season_ranking)): - tmp = friend_season_ranking[x]._asdict() - tmp.pop("user") + for row in rows[:max_ct]: + tmp = row._asdict() + tmp.pop("id") + tmp.pop("user") tmp["recordDate"] = datetime.strftime( tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0" ) + friend_season_ranking_list.append(tmp) - if len(friend_season_ranking_list) >= max_ct: - break - - if len(friend_season_ranking) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userFriendSeasonRankingList": friend_season_ranking_list, } async def handle_get_user_map_api_request(self, data: Dict) -> Dict: - maps = await self.data.item.get_maps(data["userId"]) - if maps is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_maps( + user_id, limit=max_ct + 1, offset=next_idx, + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": 0, "userMapList": [], } map_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(maps)): - tmp = maps[x]._asdict() + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("user") tmp.pop("id") map_list.append(tmp) - if len(map_list) >= max_ct: - break - - if len(maps) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userMapList": map_list, } async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict: - login_bonuses = await self.data.item.get_login_bonuses(data["userId"]) - if login_bonuses is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_login_bonuses( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { "userId": data["userId"], "nextIndex": 0, @@ -753,25 +779,20 @@ class Mai2Base: } login_bonus_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(login_bonuses)): - tmp = login_bonuses[x]._asdict() + for row in rows[:max_ct]: + tmp = row._asdict() tmp.pop("user") tmp.pop("id") login_bonus_list.append(tmp) - if len(login_bonus_list) >= max_ct: - break - - if len(login_bonuses) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userLoginBonusList": login_bonus_list, } @@ -805,42 +826,54 @@ class Mai2Base: return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []} async def handle_get_user_music_api_request(self, data: Dict) -> Dict: - user_id = data.get("userId", 0) - next_index = data.get("nextIndex", 0) - max_ct = data.get("maxCount", 50) - upper_lim = next_index + max_ct - music_detail_list = [] + user_id: int = data.get("userId", 0) + next_idx: int = data.get("nextIndex", 0) + max_ct: int = data.get("maxCount", 50) if user_id <= 0: self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0") return {} - songs = await self.data.score.get_best_scores(user_id, is_dx=False) - if songs is None: + rows = await self.data.score.get_best_scores( + user_id, is_dx=False, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!") return { - "userId": data["userId"], - "nextIndex": 0, - "userMusicList": [], - } + "userId": user_id, + "nextIndex": 0, + "userMusicList": [], + } - num_user_songs = len(songs) + music_details = [row._asdict() for row in rows] + returned_count = 0 + music_list = [] - for x in range(next_index, upper_lim): - if num_user_songs <= x: + for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]): + details: list[dict[Any, Any]] = [] + + for d in details_iter: + d.pop("id") + d.pop("user") + + details.append(d) + + music_list.append({"userMusicDetailList": details}) + returned_count += len(details) + + if len(music_list) >= max_ct: break + + if returned_count < len(rows): + next_idx += max_ct + else: + next_idx = 0 - tmp = songs[x]._asdict() - tmp.pop("id") - tmp.pop("user") - music_detail_list.append(tmp) - - next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim - self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})") return { - "userId": data["userId"], - "nextIndex": next_index, - "userMusicList": [{"userMusicDetailList": music_detail_list}], + "userId": user_id, + "nextIndex": next_idx, + "userMusicList": music_list, } async def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict: @@ -925,30 +958,52 @@ class Mai2Base: async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict: user_id = data.get("userId", 0) kind = data.get("kind", 0) # 1 is fav music, 2 is rival user IDs - next_index = data.get("nextIndex", 0) + next_idx = data.get("nextIndex", 0) max_ct = data.get("maxCount", 100) # always 100 is_all = data.get("isAllFavoriteItem", False) # always false + + empty_resp = { + "userId": user_id, + "kind": kind, + "nextIndex": 0, + "userFavoriteItemList": [], + } + + if not user_id or kind not in (1, 2): + return empty_resp + id_list: List[Dict] = [] - if user_id: - if kind == 1: - fav_music = await self.data.item.get_fav_music(user_id) - if fav_music: - for fav in fav_music: - id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]}) - if len(id_list) >= 100: # Lazy but whatever - break - - elif kind == 2: - rivals = await self.data.profile.get_rivals_game(user_id) - if rivals: - for rival in rivals: - id_list.append({"orderId": 0, "id": rival["rival"]}) + if kind == 1: + rows = await self.data.item.get_fav_music( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: + return empty_resp + + for row in rows[:max_ct]: + id_list.append({"orderId": row["orderId"] or 0, "id": row["musicId"]}) + elif kind == 2: + rows = await self.data.profile.get_rivals_game( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: + return empty_resp + + for row in rows[:max_ct]: + id_list.append({"orderId": 0, "id": row["rival"]}) + + if rows is None or len(rows) <= max_ct: + next_idx = 0 + else: + next_idx += max_ct return { "userId": user_id, "kind": kind, - "nextIndex": 0, + "nextIndex": next_idx, "userFavoriteItemList": id_list, } @@ -964,5 +1019,4 @@ class Mai2Base: """ return {"userId": data["userId"], "userRecommendSelectionMusicIdList": []} async def handle_get_user_score_ranking_api_request(self, data: Dict) ->Dict: - - return {"userId": data["userId"], "userScoreRanking": []} \ No newline at end of file + return {"userId": data["userId"], "userScoreRanking": []} diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 66bf914..b37a3f4 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -1,8 +1,9 @@ -from typing import Any, List, Dict +import itertools from datetime import datetime, timedelta -import pytz -import json from random import randint +from typing import Any, Dict, List + +import pytz from core.config import CoreConfig from core.utils import Utils @@ -309,83 +310,112 @@ class Mai2DX(Mai2Base): return {"userId": data["userId"], "userOption": options_dict} async def handle_get_user_card_api_request(self, data: Dict) -> Dict: - user_cards = await self.data.item.get_cards(data["userId"]) - if user_cards is None: - return {"userId": data["userId"], "nextIndex": 0, "userCardList": []} + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_cards(user_id, limit=max_ct + 1, offset=next_idx) + + if rows is None: + return {"userId": user_id, "nextIndex": 0, "userCardList": []} - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx + card_list = [] - if len(user_cards[start_idx:]) > max_ct: + for row in rows[:max_ct]: + card = row._asdict() + card.pop("id") + card.pop("user") + card["startDate"] = datetime.strftime( + card["startDate"], Mai2Constants.DATE_TIME_FORMAT + ) + card["endDate"] = datetime.strftime( + card["endDate"], Mai2Constants.DATE_TIME_FORMAT + ) + card_list.append(card) + + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 - card_list = [] - for card in user_cards: - tmp = card._asdict() - tmp.pop("id") - tmp.pop("user") - tmp["startDate"] = datetime.strftime( - tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT - ) - tmp["endDate"] = datetime.strftime( - tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT - ) - card_list.append(tmp) - return { "userId": data["userId"], "nextIndex": next_idx, - "userCardList": card_list[start_idx:end_idx], + "userCardList": card_list, } async def handle_get_user_item_api_request(self, data: Dict) -> Dict: - kind = data["nextIndex"] // 10000000000 - next_idx = data["nextIndex"] % 10000000000 + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + kind = next_idx // 10000000000 + next_idx = next_idx % 10000000000 + items: List[Dict[str, Any]] = [] if kind == 4: # presents - user_pres_list = await self.data.item.get_presents_by_version_user(self.version, data["userId"]) - if user_pres_list: - self.logger.debug(f"Found {len(user_pres_list)} possible presents") - for present in user_pres_list: - if (present['startDate'] and present['startDate'].timestamp() > datetime.now().timestamp()): - self.logger.debug(f"Present {present['id']} distribution hasn't started yet (begins {present['startDate']})") - continue # present period hasn't started yet, move onto the next one - - if (present['endDate'] and present['endDate'].timestamp() < datetime.now().timestamp()): - self.logger.warn(f"Present {present['id']} ended on {present['endDate']} and should be removed") - continue # present period ended, move onto the next one - - test = await self.data.item.get_item(data["userId"], present['itemKind'], present['itemId']) - if not test: # Don't send presents for items the user already has - pres_id = present['itemKind'] * 1000000 - pres_id += present['itemId'] - items.append({"itemId": pres_id, "itemKind": 4, "stock": present['stock'], "isValid": True}) - self.logger.info(f"Give user {data['userId']} {present['stock']}x item {present['itemId']} (kind {present['itemKind']}) as present") + rows = await self.data.item.get_presents_by_version_user( + version=self.version, + user_id=user_id, + exclude_owned=True, + exclude_not_in_present_period=True, + limit=max_ct + 1, + offset=next_idx, + ) + if rows is None: + return { + "userId": user_id, + "nextIndex": 0, + "itemKind": kind, + "userItemList": [], + } + + for row in rows[:max_ct]: + self.logger.info( + f"Give user {user_id} {row['stock']}x item {row['itemId']} (kind {row['itemKind']}) as present" + ) + + items.append( + { + "itemId": row["itemKind"] * 1000000 + row["itemId"], + "itemKind": kind, + "stock": row["stock"], + "isValid": True, + } + ) else: - user_item_list = await self.data.item.get_items(data["userId"], kind) - for i in range(next_idx, len(user_item_list)): - tmp = user_item_list[i]._asdict() - tmp.pop("user") - tmp.pop("id") - items.append(tmp) - if len(items) >= int(data["maxCount"]): - break + rows = await self.data.item.get_items( + user_id=user_id, + item_kind=kind, + limit=max_ct + 1, + offset=next_idx, + ) - xout = kind * 10000000000 + next_idx + len(items) + if rows is None: + return { + "userId": user_id, + "nextIndex": 0, + "itemKind": kind, + "userItemList": [], + } - if len(items) < int(data["maxCount"]): + for row in rows[:max_ct]: + item = row._asdict() + + item.pop("id") + item.pop("user") + + items.append(item) + + if len(rows) > max_ct: + next_idx = kind * 10000000000 + next_idx + max_ct + else: next_idx = 0 - else: - next_idx = xout return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "itemKind": kind, "userItemList": items, @@ -491,103 +521,115 @@ class Mai2DX(Mai2Base): return {"length": 0, "userPortraitList": []} async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict: - friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"]) - if friend_season_ranking is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_friend_season_ranking( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": 0, "userFriendSeasonRankingList": [], } friend_season_ranking_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(friend_season_ranking)): - tmp = friend_season_ranking[x]._asdict() - tmp.pop("user") - tmp.pop("id") - tmp["recordDate"] = datetime.strftime( - tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0" + for row in rows[:max_ct]: + friend_season_ranking = row._asdict() + + friend_season_ranking.pop("user") + friend_season_ranking.pop("id") + friend_season_ranking["recordDate"] = datetime.strftime( + friend_season_ranking["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0" ) - friend_season_ranking_list.append(tmp) + + friend_season_ranking_list.append(friend_season_ranking) - if len(friend_season_ranking_list) >= max_ct: - break - - if len(friend_season_ranking) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userFriendSeasonRankingList": friend_season_ranking_list, } async def handle_get_user_map_api_request(self, data: Dict) -> Dict: - maps = await self.data.item.get_maps(data["userId"]) - if maps is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_maps( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": 0, "userMapList": [], } map_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(maps)): - tmp = maps[x]._asdict() - tmp.pop("user") - tmp.pop("id") - map_list.append(tmp) + for row in rows[:max_ct]: + map = row._asdict() + + map.pop("user") + map.pop("id") + + map_list.append(map) - if len(map_list) >= max_ct: - break - - if len(maps) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userMapList": map_list, } async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict: - login_bonuses = await self.data.item.get_login_bonuses(data["userId"]) - if login_bonuses is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_login_bonuses( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "nextIndex": 0, "userLoginBonusList": [], } login_bonus_list = [] - next_idx = int(data["nextIndex"]) - max_ct = int(data["maxCount"]) - for x in range(next_idx, len(login_bonuses)): - tmp = login_bonuses[x]._asdict() - tmp.pop("user") - tmp.pop("id") - login_bonus_list.append(tmp) + for row in rows[:max_ct]: + login_bonus = row._asdict() + + login_bonus.pop("user") + login_bonus.pop("id") + + login_bonus_list.append(login_bonus) - if len(login_bonus_list) >= max_ct: - break - - if len(login_bonuses) >= next_idx + max_ct: + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 return { - "userId": data["userId"], + "userId": user_id, "nextIndex": next_idx, "userLoginBonusList": login_bonus_list, } @@ -619,46 +661,62 @@ class Mai2DX(Mai2Base): } async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict: - user_id = data.get("userId", 0) - rival_id = data.get("rivalId", 0) - next_index = data.get("nextIndex", 0) - max_ct = 100 - upper_lim = next_index + max_ct - rival_music_list: Dict[int, List] = {} + user_id: int = data["userId"] + rival_id: int = data["rivalId"] + next_idx: int = data["nextIndex"] + max_ct: int = 100 + levels: list[int] = [x["level"] for x in data["userRivalMusicLevelList"]] - songs = await self.data.score.get_best_scores(rival_id) - if songs is None: + rows = await self.data.score.get_best_scores( + rival_id, + is_dx=True, + limit=max_ct + 1, + offset=next_idx, + levels=levels, + ) + + if rows is None: self.logger.debug("handle_get_user_rival_music_api_request: get_best_scores returned None!") + return { "userId": user_id, "rivalId": rival_id, "nextIndex": 0, "userRivalMusicList": [] # musicId userRivalMusicDetailList -> level achievement deluxscoreMax } + + music_details = [x._asdict() for x in rows] + returned_count = 0 + music_list = [] - num_user_songs = len(songs) + for music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]): + details: list[dict[Any, Any]] = [] - for x in range(next_index, upper_lim): - if x >= num_user_songs: + for d in details_iter: + details.append( + { + "level": d["level"], + "achievement": d["achievement"], + "deluxscoreMax": d["deluxscoreMax"], + } + ) + + music_list.append({"musicId": music_id, "userRivalMusicDetailList": details}) + returned_count += len(details) + + if len(music_list) >= max_ct: break - tmp = songs[x]._asdict() - if tmp['musicId'] in rival_music_list: - rival_music_list[tmp['musicId']].append([{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}]) - - else: - if len(rival_music_list) >= max_ct: - break - rival_music_list[tmp['musicId']] = [{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}] - - next_index = 0 if len(rival_music_list) < max_ct or num_user_songs == upper_lim else upper_lim - self.logger.info(f"Send rival {rival_id} songs {next_index}-{upper_lim} ({len(rival_music_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})") + if returned_count < len(rows): + next_idx += max_ct + else: + next_idx = 0 return { "userId": user_id, "rivalId": rival_id, - "nextIndex": next_index, - "userRivalMusicList": [{"musicId": x, "userRivalMusicDetailList": y} for x, y in rival_music_list.items()] + "nextIndex": next_idx, + "userRivalMusicList": music_list, } async def handle_get_user_new_item_api_request(self, data: Dict) -> Dict: @@ -674,42 +732,55 @@ class Mai2DX(Mai2Base): } async def handle_get_user_music_api_request(self, data: Dict) -> Dict: - user_id = data.get("userId", 0) - next_index = data.get("nextIndex", 0) - max_ct = data.get("maxCount", 50) - upper_lim = next_index + max_ct - music_detail_list = [] + user_id: int = data.get("userId", 0) + next_idx: int = data.get("nextIndex", 0) + max_ct: int = data.get("maxCount", 50) if user_id <= 0: self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0") return {} - songs = await self.data.score.get_best_scores(user_id) - if songs is None: + rows = await self.data.score.get_best_scores( + user_id, is_dx=True, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!") + return { - "userId": data["userId"], - "nextIndex": 0, - "userMusicList": [], - } + "userId": user_id, + "nextIndex": 0, + "userMusicList": [], + } - num_user_songs = len(songs) + music_details = [row._asdict() for row in rows] + returned_count = 0 + music_list = [] - for x in range(next_index, upper_lim): - if num_user_songs <= x: + for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]): + details: list[dict[Any, Any]] = [] + + for d in details_iter: + d.pop("id") + d.pop("user") + + details.append(d) + + music_list.append({"userMusicDetailList": details}) + returned_count += len(details) + + if len(music_list) >= max_ct: break + + if returned_count < len(rows): + next_idx += max_ct + else: + next_idx = 0 - tmp = songs[x]._asdict() - tmp.pop("id") - tmp.pop("user") - music_detail_list.append(tmp) - - next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim - self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})") return { - "userId": data["userId"], - "nextIndex": next_index, - "userMusicList": [{"userMusicDetailList": music_detail_list}], + "userId": user_id, + "nextIndex": next_idx, + "userMusicList": music_list, } async def handle_user_login_api_request(self, data: Dict) -> Dict: @@ -812,39 +883,43 @@ class Mai2DX(Mai2Base): return {"length": len(selling_card_list), "sellingCardList": selling_card_list} async def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict: - user_cards = await self.data.item.get_cards(data["userId"]) - if user_cards is None: + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.item.get_cards( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return {"returnCode": 1, "length": 0, "nextIndex": 0, "userCardList": []} - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx + card_list = [] - if len(user_cards[start_idx:]) > max_ct: + for row in rows[:max_ct]: + card = row._asdict() + + card.pop("id") + card.pop("user") + card["startDate"] = datetime.strftime( + card["startDate"], Mai2Constants.DATE_TIME_FORMAT + ) + card["endDate"] = datetime.strftime( + card["endDate"], Mai2Constants.DATE_TIME_FORMAT + ) + + card_list.append(card) + + if len(rows) > max_ct: next_idx += max_ct else: next_idx = 0 - card_list = [] - for card in user_cards: - tmp = card._asdict() - tmp.pop("id") - tmp.pop("user") - - tmp["startDate"] = datetime.strftime( - tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT - ) - tmp["endDate"] = datetime.strftime( - tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT - ) - card_list.append(tmp) - return { "returnCode": 1, - "length": len(card_list[start_idx:end_idx]), + "length": len(card_list), "nextIndex": next_idx, - "userCardList": card_list[start_idx:end_idx], + "userCardList": card_list, } async def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict: diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index 87ddca4..3b7d8d4 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -1,15 +1,16 @@ -from core.data.schema import BaseData, metadata - from datetime import datetime -from typing import Optional, Dict, List -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, or_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BIGINT, INTEGER -from sqlalchemy.schema import ForeignKey -from sqlalchemy.sql import func, select +from typing import Dict, List, Optional + +from sqlalchemy import Column, Table, UniqueConstraint, and_, or_ from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row +from sqlalchemy.schema import ForeignKey +from sqlalchemy.sql import func, select +from sqlalchemy.types import BIGINT, INTEGER, JSON, TIMESTAMP, Boolean, Integer, String -character = Table( +from core.data.schema import BaseData, metadata + +character: Table = Table( "mai2_item_character", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -27,7 +28,7 @@ character = Table( mysql_charset="utf8mb4", ) -card = Table( +card: Table = Table( "mai2_item_card", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -46,7 +47,7 @@ card = Table( mysql_charset="utf8mb4", ) -item = Table( +item: Table = Table( "mai2_item_item", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -63,7 +64,7 @@ item = Table( mysql_charset="utf8mb4", ) -map = Table( +map: Table = Table( "mai2_item_map", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -81,7 +82,7 @@ map = Table( mysql_charset="utf8mb4", ) -login_bonus = Table( +login_bonus: Table = Table( "mai2_item_login_bonus", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -98,7 +99,7 @@ login_bonus = Table( mysql_charset="utf8mb4", ) -friend_season_ranking = Table( +friend_season_ranking: Table = Table( "mai2_item_friend_season_ranking", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -134,7 +135,7 @@ favorite = Table( mysql_charset="utf8mb4", ) -fav_music = Table( +fav_music: Table = Table( "mai2_item_favorite_music", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -199,7 +200,7 @@ print_detail = Table( mysql_charset="utf8mb4", ) -present = Table( +present: Table = Table( "mai2_item_present", metadata, Column('id', BIGINT, primary_key=True, nullable=False), @@ -239,13 +240,26 @@ class Mai2ItemData(BaseData): return None return result.lastrowid - async def get_items(self, user_id: int, item_kind: int = None) -> Optional[List[Row]]: - if item_kind is None: - sql = item.select(item.c.user == user_id) - else: - sql = item.select( - and_(item.c.user == user_id, item.c.itemKind == item_kind) - ) + async def get_items( + self, + user_id: int, + item_kind: Optional[int] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + cond = item.c.user == user_id + + if item_kind is not None: + cond &= item.c.itemKind == item_kind + + sql = select(item).where(cond) + + if limit is not None or offset is not None: + sql = sql.order_by(item.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -296,8 +310,20 @@ class Mai2ItemData(BaseData): return None return result.lastrowid - async def get_login_bonuses(self, user_id: int) -> Optional[List[Row]]: - sql = login_bonus.select(login_bonus.c.user == user_id) + async def get_login_bonuses( + self, + user_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(login_bonus).where(login_bonus.c.user == user_id) + + if limit is not None or offset is not None: + sql = sql.order_by(login_bonus.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -347,8 +373,20 @@ class Mai2ItemData(BaseData): return None return result.lastrowid - async def get_maps(self, user_id: int) -> Optional[List[Row]]: - sql = map.select(map.c.user == user_id) + async def get_maps( + self, + user_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(map).where(map.c.user == user_id) + + if limit is not None or offset is not None: + sql = sql.order_by(map.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -424,8 +462,20 @@ class Mai2ItemData(BaseData): return None return result.fetchone() - async def get_friend_season_ranking(self, user_id: int) -> Optional[Row]: - sql = friend_season_ranking.select(friend_season_ranking.c.user == user_id) + async def get_friend_season_ranking( + self, + user_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(friend_season_ranking).where(friend_season_ranking.c.user == user_id) + + if limit is not None or offset is not None: + sql = sql.order_by(friend_season_ranking.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -480,8 +530,23 @@ class Mai2ItemData(BaseData): return None return result.fetchall() - async def get_fav_music(self, user_id: int) -> Optional[List[Row]]: - result = await self.execute(fav_music.select(fav_music.c.user == user_id)) + async def get_fav_music( + self, + user_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(fav_music).where(fav_music.c.user == user_id) + + if limit is not None or offset is not None: + sql = sql.order_by(fav_music.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + + result = await self.execute(sql) + if result: return result.fetchall() @@ -537,13 +602,24 @@ class Mai2ItemData(BaseData): return None return result.lastrowid - async def get_cards(self, user_id: int, kind: int = None) -> Optional[Row]: - if kind is None: - sql = card.select(card.c.user == user_id) - else: - sql = card.select(and_(card.c.user == user_id, card.c.cardKind == kind)) + async def get_cards( + self, + user_id: int, + kind: Optional[int] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + condition = card.c.user == user_id - sql = sql.order_by(card.c.startDate.desc()) + if kind is not None: + condition &= card.c.cardKind == kind + + sql = select(card).where(condition).order_by(card.c.startDate.desc(), card.c.id.asc()) + + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: @@ -634,13 +710,46 @@ class Mai2ItemData(BaseData): if result: return result.fetchall() - async def get_presents_by_version_user(self, ver: int = None, user_id: int = None) -> Optional[List[Row]]: - result = await self.execute(present.select( - and_( - or_(present.c.user == user_id, present.c.user == None), - or_(present.c.version == ver, present.c.version == None) + async def get_presents_by_version_user( + self, + version: Optional[int] = None, + user_id: Optional[int] = None, + exclude_owned: bool = False, + exclude_not_in_present_period: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(present) + condition = ( + ((present.c.user == user_id) | present.c.user.is_(None)) + & ((present.c.version == version) | present.c.version.is_(None)) + ) + + # Do an anti-join with the mai2_item_item table to exclude any + # items the users have already owned. + if exclude_owned: + sql = sql.join( + item, + (present.c.itemKind == item.c.itemKind) + & (present.c.itemId == item.c.itemId) ) - )) + condition &= (item.c.itemKind.is_(None) & item.c.itemId.is_(None)) + + if exclude_not_in_present_period: + condition &= (present.c.startDate.is_(None) | (present.c.startDate <= func.now())) + condition &= (present.c.endDate.is_(None) | (present.c.endDate >= func.now())) + + sql = sql.where(condition) + + if limit is not None or offset is not None: + sql = sql.order_by(present.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + + result = await self.execute(sql) + if result: return result.fetchall() diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index ede0adf..ebf04a4 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -1,15 +1,26 @@ -from core.data.schema import BaseData, metadata -from titles.mai2.const import Mai2Constants +from datetime import datetime +from typing import Dict, List, Optional from uuid import uuid4 -from typing import Optional, Dict, List -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER +from sqlalchemy import Column, Table, UniqueConstraint, and_ +from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import Row from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select -from sqlalchemy.engine import Row -from sqlalchemy.dialects.mysql import insert -from datetime import datetime +from sqlalchemy.types import ( + INTEGER, + JSON, + TIMESTAMP, + VARCHAR, + BigInteger, + Boolean, + Integer, + SmallInteger, + String, +) + +from core.data.schema import BaseData, metadata +from titles.mai2.const import Mai2Constants detail = Table( "mai2_profile_detail", @@ -495,7 +506,7 @@ consec_logins = Table( mysql_charset="utf8mb4", ) -rival = Table( +rival: Table = Table( "mai2_user_rival", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -908,8 +919,23 @@ class Mai2ProfileData(BaseData): if result: return result.fetchall() - async def get_rivals_game(self, user_id: int) -> Optional[List[Row]]: - result = await self.execute(rival.select(and_(rival.c.user == user_id, rival.c.show == True)).limit(3)) + async def get_rivals_game( + self, + user_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + sql = select(rival).where((rival.c.user == user_id) & rival.c.show.is_(True)) + + if limit is not None or offset is not None: + sql = sql.order_by(rival.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + + result = await self.execute(sql) + if result: return result.fetchall() diff --git a/titles/mai2/schema/score.py b/titles/mai2/schema/score.py index e376216..cbe7448 100644 --- a/titles/mai2/schema/score.py +++ b/titles/mai2/schema/score.py @@ -1,15 +1,15 @@ from typing import Dict, List, Optional -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger + +from sqlalchemy import Column, Table, UniqueConstraint, and_ +from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import Row from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select -from sqlalchemy.engine import Row -from sqlalchemy.dialects.mysql import insert +from sqlalchemy.types import JSON, BigInteger, Boolean, Integer, String from core.data.schema import BaseData, metadata -from core.data import cached -best_score = Table( +best_score: Table = Table( "mai2_score_best", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -272,7 +272,7 @@ playlog_old = Table( mysql_charset="utf8mb4", ) -best_score_old = Table( +best_score_old: Table = Table( "maimai_score_best", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -313,22 +313,55 @@ class Mai2ScoreData(BaseData): return None return result.lastrowid - @cached(2) - async def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]: + async def get_best_scores( + self, + user_id: int, + song_id: Optional[int] = None, + is_dx: bool = True, + limit: Optional[int] = None, + offset: Optional[int] = None, + levels: Optional[list[int]] = None, + ) -> Optional[List[Row]]: if is_dx: - sql = best_score.select( - and_( - best_score.c.user == user_id, - (best_score.c.musicId == song_id) if song_id is not None else True, - ) - ).order_by(best_score.c.musicId).order_by(best_score.c.level) + table = best_score else: - sql = best_score_old.select( - and_( - best_score_old.c.user == user_id, - (best_score_old.c.musicId == song_id) if song_id is not None else True, - ) - ).order_by(best_score.c.musicId).order_by(best_score.c.level) + table = best_score_old + + cond = table.c.user == user_id + + if song_id is not None: + cond &= table.c.musicId == song_id + + if levels is not None: + cond &= table.c.level.in_(levels) + + if limit is None and offset is None: + sql = ( + select(table) + .where(cond) + .order_by(table.c.musicId, table.c.level) + ) + else: + subq = ( + select(table.c.musicId) + .distinct() + .where(cond) + .order_by(table.c.musicId) + ) + + if limit is not None: + subq = subq.limit(limit) + if offset is not None: + subq = subq.offset(offset) + + subq = subq.subquery() + + sql = ( + select(table) + .join(subq, table.c.musicId == subq.c.musicId) + .where(cond) + .order_by(table.c.musicId, table.c.level) + ) result = await self.execute(sql) if result is None: diff --git a/titles/ongeki/base.py b/titles/ongeki/base.py index e454dc3..1bebb4d 100644 --- a/titles/ongeki/base.py +++ b/titles/ongeki/base.py @@ -1,16 +1,16 @@ -from datetime import date, datetime, timedelta -from typing import Any, Dict, List +import itertools import json import logging +from datetime import datetime, timedelta from enum import Enum +from typing import Any, Dict, List import pytz + from core.config import CoreConfig -from core.data.cache import cached +from titles.ongeki.config import OngekiConfig from titles.ongeki.const import OngekiConstants -from titles.ongeki.config import OngekiConfig from titles.ongeki.database import OngekiData -from titles.ongeki.config import OngekiConfig class OngekiBattleGrade(Enum): @@ -500,57 +500,93 @@ class OngekiBase: } async def handle_get_user_music_api_request(self, data: Dict) -> Dict: - song_list = await self.util_generate_music_list(data["userId"]) - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] - if len(song_list[start_idx:]) > max_ct: + rows = await self.data.score.get_best_scores( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: + return { + "userId": user_id, + "length": 0, + "nextIndex": 0, + "userMusicList": [], + } + + music_details = [row._asdict() for row in rows] + returned_count = 0 + music_list = [] + + for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]): + details: list[dict[Any, Any]] = [] + + for d in details_iter: + d.pop("id") + d.pop("user") + + details.append(d) + + music_list.append({"length": len(details), "userMusicDetailList": details}) + returned_count += len(details) + + if len(music_list) >= max_ct: + break + + if returned_count < len(rows): next_idx += max_ct - else: - next_idx = -1 + next_idx = 0 return { - "userId": data["userId"], - "length": len(song_list[start_idx:end_idx]), + "userId": user_id, + "length": len(music_list), "nextIndex": next_idx, - "userMusicList": song_list[start_idx:end_idx], + "userMusicList": music_list, } async def handle_get_user_item_api_request(self, data: Dict) -> Dict: - kind = data["nextIndex"] / 10000000000 - p = await self.data.item.get_items(data["userId"], kind) + user_id: int = data["userId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] - if p is None: + kind = next_idx // 10000000000 + next_idx = next_idx % 10000000000 + + rows = await self.data.item.get_items( + user_id, kind, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], - "nextIndex": -1, + "userId": user_id, + "nextIndex": 0, "itemKind": kind, + "length": 0, "userItemList": [], } items: List[Dict[str, Any]] = [] - for i in range(data["nextIndex"] % 10000000000, len(p)): - if len(items) > data["maxCount"]: - break - tmp = p[i]._asdict() - tmp.pop("user") - tmp.pop("id") - items.append(tmp) - xout = kind * 10000000000 + (data["nextIndex"] % 10000000000) + len(items) + for row in rows[:max_ct]: + item = row._asdict() + + item.pop("id") + item.pop("user") + + items.append(item) - if len(items) < data["maxCount"] or data["maxCount"] == 0: - nextIndex = 0 + if len(rows) > max_ct: + next_idx = kind * 10000000000 + next_idx + max_ct else: - nextIndex = xout + next_idx = 0 return { - "userId": data["userId"], - "nextIndex": int(nextIndex), - "itemKind": int(kind), + "userId": user_id, + "nextIndex": next_idx, + "itemKind": kind, "length": len(items), "userItemList": items, } @@ -1143,43 +1179,56 @@ class OngekiBase: """ Added in Bright """ - rival_id = data["rivalUserId"] - next_idx = data["nextIndex"] - max_ct = data["maxCount"] - music = self.handle_get_user_music_api_request( - {"userId": rival_id, "nextIndex": next_idx, "maxCount": max_ct} + user_id: int = data["userId"] + rival_id: int = data["rivalUserId"] + next_idx: int = data["nextIndex"] + max_ct: int = data["maxCount"] + + rows = await self.data.score.get_best_scores( + rival_id, limit=max_ct + 1, offset=next_idx ) - for song in music["userMusicList"]: - song["userRivalMusicDetailList"] = song["userMusicDetailList"] - song.pop("userMusicDetailList") + if rows is None: + return { + "userId": user_id, + "rivalUserId": rival_id, + "nextIndex": 0, + "length": 0, + "userRivalMusicList": [], + } + + music_details = [row._asdict() for row in rows] + returned_count = 0 + music_list = [] + + for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]): + details: list[dict[Any, Any]] = [] + + for d in details_iter: + d.pop("id") + d.pop("user") + d.pop("playCount") + d.pop("isLock") + d.pop("clearStatus") + d.pop("isStoryWatched") + + details.append(d) + + music_list.append({"length": len(details), "userRivalMusicDetailList": details}) + returned_count += len(details) + + if len(music_list) >= max_ct: + break + + if returned_count < len(rows): + next_idx += max_ct + else: + next_idx = 0 + return { - "userId": data["userId"], + "userId": user_id, "rivalUserId": rival_id, - "length": music["length"], - "nextIndex": music["nextIndex"], - "userRivalMusicList": music["userMusicList"], + "nextIndex": next_idx, + "length": len(music_list), + "userRivalMusicList": music_list, } - - @cached(2) - async def util_generate_music_list(self, user_id: int) -> List: - music_detail = await self.data.score.get_best_scores(user_id) - song_list = [] - - for md in music_detail: - found = False - tmp = md._asdict() - tmp.pop("user") - tmp.pop("id") - - for song in song_list: - if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]: - found = True - song["userMusicDetailList"].append(tmp) - song["length"] = len(song["userMusicDetailList"]) - break - - if not found: - song_list.append({"length": 1, "userMusicDetailList": [tmp]}) - - return song_list diff --git a/titles/ongeki/bright.py b/titles/ongeki/bright.py index 690a118..5c95af3 100644 --- a/titles/ongeki/bright.py +++ b/titles/ongeki/bright.py @@ -1,13 +1,11 @@ -from datetime import date, datetime, timedelta -from typing import Any, Dict +from datetime import datetime from random import randint -import pytz -import json +from typing import Dict from core.config import CoreConfig from titles.ongeki.base import OngekiBase -from titles.ongeki.const import OngekiConstants from titles.ongeki.config import OngekiConfig +from titles.ongeki.const import OngekiConstants class OngekiBright(OngekiBase): @@ -62,66 +60,72 @@ class OngekiBright(OngekiBase): return {"returnCode": 1} async def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict: - user_cards = await self.data.item.get_cards(data["userId"]) - if user_cards is None: + user_id: int = data["userId"] + max_ct: int = data["maxCount"] + next_idx: int = data["nextIndex"] + + rows = await self.data.item.get_cards( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return {} + + card_list = [] - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx - - if len(user_cards[start_idx:]) > max_ct: + for row in rows[:max_ct]: + card = row._asdict() + card.pop("id") + card.pop("user") + card_list.append(card) + + if len(rows) > max_ct: next_idx += max_ct else: - next_idx = -1 - - card_list = [] - for card in user_cards: - tmp = card._asdict() - tmp.pop("id") - tmp.pop("user") - card_list.append(tmp) + next_idx = 0 return { "userId": data["userId"], - "length": len(card_list[start_idx:end_idx]), + "length": len(card_list), "nextIndex": next_idx, - "userCardList": card_list[start_idx:end_idx], + "userCardList": card_list, } async def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict: - user_characters = await self.data.item.get_characters(data["userId"]) - if user_characters is None: + user_id: int = data["userId"] + max_ct: int = data["maxCount"] + next_idx: int = data["nextIndex"] + + rows = await self.data.item.get_characters( + user_id, limit=max_ct + 1, offset=next_idx + ) + + if rows is None: return { - "userId": data["userId"], + "userId": user_id, "length": 0, "nextIndex": 0, "userCharacterList": [], } - max_ct = data["maxCount"] - next_idx = data["nextIndex"] - start_idx = next_idx - end_idx = max_ct + start_idx + character_list = [] - if len(user_characters[start_idx:]) > max_ct: + for row in rows[:max_ct]: + character = row._asdict() + character.pop("id") + character.pop("user") + character_list.append(character) + + if len(rows) > max_ct: next_idx += max_ct else: - next_idx = -1 - - character_list = [] - for character in user_characters: - tmp = character._asdict() - tmp.pop("id") - tmp.pop("user") - character_list.append(tmp) + next_idx = 0 return { "userId": data["userId"], - "length": len(character_list[start_idx:end_idx]), + "length": len(character_list), "nextIndex": next_idx, - "userCharacterList": character_list[start_idx:end_idx], + "userCharacterList": character_list, } async def handle_get_user_gacha_api_request(self, data: Dict) -> Dict: diff --git a/titles/ongeki/schema/item.py b/titles/ongeki/schema/item.py index ca2de1f..274e16d 100644 --- a/titles/ongeki/schema/item.py +++ b/titles/ongeki/schema/item.py @@ -1,15 +1,16 @@ -from datetime import date, datetime, timedelta -from typing import Dict, Optional, List -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON -from sqlalchemy.schema import ForeignKey -from sqlalchemy.engine import Row -from sqlalchemy.sql import func, select +from datetime import datetime +from typing import Dict, List, Optional + +from sqlalchemy import Column, Table, UniqueConstraint, and_ from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import Row +from sqlalchemy.schema import ForeignKey +from sqlalchemy.sql import func, select +from sqlalchemy.types import TIMESTAMP, Boolean, Integer, String from core.data.schema import BaseData, metadata -card = Table( +card: Table = Table( "ongeki_user_card", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -45,7 +46,7 @@ deck = Table( mysql_charset="utf8mb4", ) -character = Table( +character: Table = Table( "ongeki_user_character", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -130,7 +131,7 @@ memorychapter = Table( mysql_charset="utf8mb4", ) -item = Table( +item: Table = Table( "ongeki_user_item", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -351,9 +352,18 @@ class OngekiItemData(BaseData): return None return result.lastrowid - async def get_cards(self, aime_id: int) -> Optional[List[Dict]]: + async def get_cards( + self, aime_id: int, limit: Optional[int] = None, offset: Optional[int] = None + ) -> Optional[List[Row]]: sql = select(card).where(card.c.user == aime_id) + if limit is not None or offset is not None: + sql = sql.order_by(card.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None @@ -371,9 +381,18 @@ class OngekiItemData(BaseData): return None return result.lastrowid - async def get_characters(self, aime_id: int) -> Optional[List[Dict]]: + async def get_characters( + self, aime_id: int, limit: Optional[int] = None, offset: Optional[int] = None + ) -> Optional[List[Row]]: sql = select(character).where(character.c.user == aime_id) + if limit is not None or offset is not None: + sql = sql.order_by(character.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) + result = await self.execute(sql) if result is None: return None @@ -479,13 +498,26 @@ class OngekiItemData(BaseData): return None return result.fetchone() - async def get_items(self, aime_id: int, item_kind: int = None) -> Optional[List[Dict]]: - if item_kind is None: - sql = select(item).where(item.c.user == aime_id) - else: - sql = select(item).where( - and_(item.c.user == aime_id, item.c.itemKind == item_kind) - ) + async def get_items( + self, + aime_id: int, + item_kind: Optional[int] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + cond = item.c.user == aime_id + + if item_kind is not None: + cond &= item.c.itemKind == item_kind + + sql = select(item).where(cond) + + if limit is not None or offset is not None: + sql = sql.order_by(item.c.id) + if limit is not None: + sql = sql.limit(limit) + if offset is not None: + sql = sql.offset(offset) result = await self.execute(sql) if result is None: diff --git a/titles/ongeki/schema/score.py b/titles/ongeki/schema/score.py index 6867133..178cf29 100644 --- a/titles/ongeki/schema/score.py +++ b/titles/ongeki/schema/score.py @@ -1,13 +1,15 @@ from typing import Dict, List, Optional -from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float -from sqlalchemy.schema import ForeignKey -from sqlalchemy.sql import func, select + +from sqlalchemy import Column, Table, UniqueConstraint from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import Row +from sqlalchemy.schema import ForeignKey +from sqlalchemy.sql import select +from sqlalchemy.types import TIMESTAMP, Boolean, Float, Integer, String from core.data.schema import BaseData, metadata -score_best = Table( +score_best: Table = Table( "ongeki_score_best", metadata, Column("id", Integer, primary_key=True, nullable=False), @@ -149,8 +151,41 @@ class OngekiScoreData(BaseData): return None return result.lastrowid - async def get_best_scores(self, aime_id: int) -> Optional[List[Dict]]: - sql = select(score_best).where(score_best.c.user == aime_id) + async def get_best_scores( + self, + aime_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Optional[List[Row]]: + cond = score_best.c.user == aime_id + + if limit is None and offset is None: + sql = ( + select(score_best) + .where(cond) + .order_by(score_best.c.musicId, score_best.c.level) + ) + else: + subq = ( + select(score_best.c.musicId) + .distinct() + .where(cond) + .order_by(score_best.c.musicId) + ) + + if limit is not None: + subq = subq.limit(limit) + if offset is not None: + subq = subq.offset(offset) + + subq = subq.subquery() + + sql = ( + select(score_best) + .join(subq, score_best.c.musicId == subq.c.musicId) + .where(cond) + .order_by(score_best.c.musicId, score_best.c.level) + ) result = await self.execute(sql) if result is None: From 476a911df9cf042a9cda5f071d6818f008c88f0b Mon Sep 17 00:00:00 2001 From: beerpsi Date: Mon, 25 Nov 2024 20:13:51 +0700 Subject: [PATCH 062/130] [database] fix invalid transaction being left open --- core/data/database.py | 8 ++++---- core/data/schema/base.py | 36 +++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/core/data/database.py b/core/data/database.py index 170665e..bed3e79 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -1,12 +1,11 @@ import logging import os import secrets -import ssl import string import warnings from hashlib import sha256 from logging.handlers import TimedRotatingFileHandler -from typing import Any, ClassVar, Optional +from typing import ClassVar, Optional import alembic.config import bcrypt @@ -17,6 +16,7 @@ from sqlalchemy.ext.asyncio import ( AsyncSession, create_async_engine, ) +from sqlalchemy.orm import sessionmaker from core.config import CoreConfig from core.data.schema import ArcadeData, BaseData, CardData, UserData, metadata @@ -25,7 +25,7 @@ from core.utils import MISSING, Utils class Data: engine: ClassVar[AsyncEngine] = MISSING - session: ClassVar[AsyncSession] = MISSING + session: ClassVar[sessionmaker[AsyncSession]] = MISSING user: ClassVar[UserData] = MISSING arcade: ClassVar[ArcadeData] = MISSING card: ClassVar[CardData] = MISSING @@ -53,7 +53,7 @@ class Data: self.__engine = Data.engine if Data.session is MISSING: - Data.session = AsyncSession(Data.engine, expire_on_commit=False) + Data.session = sessionmaker(Data.engine, expire_on_commit=False, class_=AsyncSession) if Data.user is MISSING: Data.user = UserData(self.config, self.session) diff --git a/core/data/schema/base.py b/core/data/schema/base.py index cb44272..adacb9c 100644 --- a/core/data/schema/base.py +++ b/core/data/schema/base.py @@ -9,6 +9,7 @@ from sqlalchemy.engine import Row from sqlalchemy.engine.cursor import CursorResult from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, text from sqlalchemy.types import INTEGER, JSON, TEXT, TIMESTAMP, Integer, String @@ -38,7 +39,7 @@ event_log: Table = Table( class BaseData: - def __init__(self, cfg: CoreConfig, conn: AsyncSession) -> None: + def __init__(self, cfg: CoreConfig, conn: sessionmaker[AsyncSession]) -> None: self.config = cfg self.conn = conn self.logger = logging.getLogger("database") @@ -46,21 +47,10 @@ class BaseData: async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]: res = None - try: - self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}") - res = await self.conn.execute(text(sql), opts) - - except SQLAlchemyError as e: - self.logger.error(f"SQLAlchemy error {e}") - return None - - except UnicodeEncodeError as e: - self.logger.error(f"UnicodeEncodeError error {e}") - return None - - except Exception: + async with self.conn() as session: try: - res = await self.conn.execute(sql, opts) + self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}") + res = await session.execute(text(sql), opts) except SQLAlchemyError as e: self.logger.error(f"SQLAlchemy error {e}") @@ -71,8 +61,20 @@ class BaseData: return None except Exception: - self.logger.error(f"Unknown error") - raise + try: + res = await session.execute(sql, opts) + + except SQLAlchemyError as e: + self.logger.error(f"SQLAlchemy error {e}") + return None + + except UnicodeEncodeError as e: + self.logger.error(f"UnicodeEncodeError error {e}") + return None + + except Exception: + self.logger.error(f"Unknown error") + raise return res From 383859388e4e4c1ea50cb6052daf122d520bf5e2 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Fri, 29 Nov 2024 22:20:55 -0500 Subject: [PATCH 063/130] chuni: fix 'NoneType' object has no attribute 'split' in score.py --- titles/chuni/schema/score.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/titles/chuni/schema/score.py b/titles/chuni/schema/score.py index ab6766a..50a8f7f 100644 --- a/titles/chuni/schema/score.py +++ b/titles/chuni/schema/score.py @@ -179,7 +179,14 @@ class ChuniRomVersion(): # sort it by version number for easy iteration ChuniRomVersion.Versions = dict(sorted(all_versions.items())) - def __init__(self, rom_version: str) -> None: + def __init__(self, rom_version: Optional[str] = None) -> None: + if rom_version is None: + self.major = 0 + self.minor = 0 + self.maint = 0 + self.version = "0.00.00" + return + (major, minor, maint) = rom_version.split('.') self.major = int(major) self.minor = int(minor) @@ -343,6 +350,10 @@ class ChuniScoreData(BaseData): # for each romVersion recorded, check if it maps back the current version we are operating on matching_rom_versions = [] for v in record_versions: + # Do this to prevent null romVersion from causing an error in ChuniRomVersion.__init__() + if v[0] is None: + continue + if ChuniRomVersion(v[0]).get_int_version() == version: matching_rom_versions += [v[0]] From a8f5ef15503e072982f894084870aa1136815ed9 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 1 Dec 2024 14:19:55 -0500 Subject: [PATCH 064/130] allnet: properly dfi encode downloadorder responses --- core/allnet.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 9eb6595..43dde1f 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -349,12 +349,22 @@ class AllnetServlet: not self.config.allnet.allow_online_updates or not self.config.allnet.update_cfg_folder ): - return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n") + resp = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n" + if is_dfi: + return PlainTextResponse( + self.to_dfi(resp) + b"\r\n", headers={ "Pragma": "DFI" } + ) + return PlainTextResponse(resp) else: machine = await self.data.arcade.get_machine(req.serial) if not machine or not machine['ota_enable'] or not machine['is_cab'] or machine['is_blacklisted']: - return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n") + resp = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n" + if is_dfi: + return PlainTextResponse( + self.to_dfi(resp) + b"\r\n", headers={ "Pragma": "DFI" } + ) + return PlainTextResponse(resp) if path.exists( f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini" @@ -744,7 +754,7 @@ class AllnetDownloadOrderRequest: self.encode = req.get("encode", "") class AllnetDownloadOrderResponse: - def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None: + def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None: self.stat = stat self.serial = serial self.uri = uri From 5ecc7984c7bee3c05ba2d4b7762b18e253588672 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Sun, 8 Dec 2024 08:44:32 +0800 Subject: [PATCH 065/130] Fix: AimeDB Felica LookupEx rename package parameter --- core/adb_handlers/felica.py | 4 ++-- core/aimedb.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/adb_handlers/felica.py b/core/adb_handlers/felica.py index b7fdc2e..a037bbe 100644 --- a/core/adb_handlers/felica.py +++ b/core/adb_handlers/felica.py @@ -39,10 +39,10 @@ class ADBFelicaLookupExRequest(ADBBaseRequest): def __init__(self, data: bytes) -> None: super().__init__(data) self.random = struct.unpack_from("<16s", data, 0x20)[0] - idm, pmm = struct.unpack_from(">QQ", data, 0x30) + idm, dfc = struct.unpack_from(">QQ", data, 0x30) self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40) self.idm = hex(idm)[2:].upper() - self.pmm = hex(pmm)[2:].upper() + self.dfc = hex(dfc)[2:].upper() self.company = CompanyCodes(int.from_bytes(company, 'little')) self.fw_ver = ReaderFwVer.from_byte(fw_ver) diff --git a/core/aimedb.py b/core/aimedb.py index 6d5bd57..35b4a1f 100644 --- a/core/aimedb.py +++ b/core/aimedb.py @@ -344,7 +344,7 @@ class AimedbServlette(): user_id = -1 self.logger.info( - f"idm {idm} ipm {req.pmm} -> access_code {access_code} user_id {user_id}" + f"idm {idm} dfc {req.dfc} -> access_code {access_code} user_id {user_id}" ) resp = ADBFelicaLookupExResponse.from_req(req.head, user_id, access_code) From d6d98d20cb1ef7a93fba15a1c810bbafa2ad8f77 Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 12 Dec 2024 20:47:34 +0700 Subject: [PATCH 066/130] fix: typing shenanigans --- core/data/database.py | 2 +- core/data/schema/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/data/database.py b/core/data/database.py index bed3e79..fb36ebd 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -25,7 +25,7 @@ from core.utils import MISSING, Utils class Data: engine: ClassVar[AsyncEngine] = MISSING - session: ClassVar[sessionmaker[AsyncSession]] = MISSING + session: ClassVar["sessionmaker[AsyncSession]"] = MISSING user: ClassVar[UserData] = MISSING arcade: ClassVar[ArcadeData] = MISSING card: ClassVar[CardData] = MISSING diff --git a/core/data/schema/base.py b/core/data/schema/base.py index adacb9c..80ab74b 100644 --- a/core/data/schema/base.py +++ b/core/data/schema/base.py @@ -39,7 +39,7 @@ event_log: Table = Table( class BaseData: - def __init__(self, cfg: CoreConfig, conn: sessionmaker[AsyncSession]) -> None: + def __init__(self, cfg: CoreConfig, conn: "sessionmaker[AsyncSession]") -> None: self.config = cfg self.conn = conn self.logger = logging.getLogger("database") From fe8f365d8a013909636826c0460178c4102e2fa1 Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 12 Dec 2024 20:49:39 +0700 Subject: [PATCH 067/130] [chunithm] fix rival music not showing up in game --- titles/chuni/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/chuni/base.py b/titles/chuni/base.py index 37bcbb3..5ac997c 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -465,7 +465,7 @@ class ChuniBase: for d in details_iter ] - music_list.append({"musicId": music_id, "length": len(details), "userMusicDetailList": details}) + music_list.append({"musicId": music_id, "length": len(details), "userRivalMusicDetailList": details}) returned_music_details_count += len(details) if len(music_list) >= max_ct: From 1dceff456d30961acaf0833369fd777b1dd194a7 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 15 Dec 2024 20:16:18 -0500 Subject: [PATCH 068/130] cxb: added missing r which fixes an issue on ubuntu 24.04.1 --- titles/cxb/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/cxb/index.py b/titles/cxb/index.py index 513b813..a4e7085 100644 --- a/titles/cxb/index.py +++ b/titles/cxb/index.py @@ -155,7 +155,7 @@ class CxbServlet(BaseServlet): filename = filetype_split[len(filetype_split) - 1] match = re.match( - "^([A-Za-z]*)(\d\d\d\d)$", filetype_split[len(filetype_split) - 1] + r"^([A-Za-z]*)(\d\d\d\d)$", filetype_split[len(filetype_split) - 1] ) if match: func_to_find += f"{inflection.underscore(match.group(1))}xxxx" From e8ea328e77ec12b673cc4a769a2d135bc1d601c5 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 15 Dec 2024 20:21:03 -0500 Subject: [PATCH 069/130] mai2: add `add_consec_login` call if `get_consec_login` returns None #189 --- titles/mai2/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/titles/mai2/base.py b/titles/mai2/base.py index add3ed2..e148533 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -212,6 +212,7 @@ class Mai2Base: lastLoginDate = "2017-12-05 07:00:00.0" if consec is None or not consec: + await self.data.profile.add_consec_login(data["userId"], self.version) consec_ct = 1 else: From 326b5988af8ce0172a27f95b622e95f155f25a09 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Wed, 18 Dec 2024 16:35:28 -0500 Subject: [PATCH 070/130] add half-working TUI --- readme.md | 3 + tui.py | 501 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100644 tui.py diff --git a/readme.md b/readme.md index 0564c70..e863e52 100644 --- a/readme.md +++ b/readme.md @@ -84,3 +84,6 @@ Read [Games specific info](docs/game_specific_info.md) for all supported games, ## Production guide See the [production guide](docs/prod.md) for running a production server. + +## Text User Interface +Invoke `tui.py` (with optional `-c ` parameter) for an interactive TUI to perform management actions (add, edit or delete users, cards, arcades and machines) without needing to spin up the frontend. Requires installing asciimatics via `pip install asciimatics` diff --git a/tui.py b/tui.py new file mode 100644 index 0000000..06c51e9 --- /dev/null +++ b/tui.py @@ -0,0 +1,501 @@ +#!/usr/bin/env +from typing import Optional, List +import asyncio +import argparse +from os import path, mkdir, W_OK, access +import yaml +import bcrypt +import secrets +import string +from sqlalchemy.engine import Row + +from core.data import Data +from core.config import CoreConfig + +try: + from asciimatics.widgets import Frame, Layout, Text, Button, RadioButtons, CheckBox, Divider + from asciimatics.scene import Scene + from asciimatics.screen import Screen + from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication +except: + print("Artemis TUI requires asciimatics, please install it using pip") + exit(1) + + +class State: + class SelectedUser: + def __init__(self, id: Optional[int] = None, name: Optional[str] = None): + self.id = id + self.name = name + + def __str__(self): + if self.id is not None: + return f"{self.name} ({self.id})" if self.name else f"User {self.id}" + return "None" + + def __int__(self): + return self.id if self.id else 0 + + class SelectedCard: + def __init__(self, id: Optional[int] = None, access_code: Optional[str] = None): + self.id = id + self.access_code = access_code + + def __str__(self): + if self.id is not None and self.access_code: + return f"{self.access_code} ({self.id})" + return "None" + + def __int__(self): + return self.id if self.id else 0 + + class SelectedArcade: + def __init__(self, id: Optional[int] = None, country: Optional[str] = None, name: Optional[str] = None): + self.id = id + self.country = country + self.name = name + + def __str__(self): + if self.id is not None: + return f"{self.name} ({self.country}{self.id:05d})" if self.name else f"{self.country}{self.id:05d}" + return "None" + + def __int__(self): + return self.id if self.id else 0 + + class SelectedMachine: + def __init__(self, id: Optional[int] = None, serial: Optional[str] = None): + self.id = id + self.serial = serial + + def __str__(self): + if self.id is not None: + return f"{self.serial} ({self.id})" + return "None" + + def __int__(self): + return self.id if self.id else 0 + + def __init__(self): + self.selected_user: self.SelectedUser = self.SelectedUser() + self.selected_card: self.SelectedCard = self.SelectedCard() + self.selected_arcade: self.SelectedArcade = self.SelectedArcade() + self.selected_machine: self.SelectedMachine = self.SelectedMachine() + self.last_err: str = "" + self.search_results: List[Row] = [] + self.search_type: str = "" + + def set_user(self, id: int, username: Optional[str]) -> None: + self.selected_user = self.SelectedUser(id, username) + + def clear_user(self) -> None: + self.selected_user = self.SelectedUser() + + def set_card(self, id: int, access_code: Optional[str]) -> None: + self.selected_card = self.SelectedCard(id, access_code) + + def clear_card(self) -> None: + self.selected_card = self.SelectedCard() + + def set_arcade(self, id: int, country: str = "JPN", name: Optional[str] = None) -> None: + self.selected_arcade = self.SelectedArcade(id, country, name) + + def clear_arcade(self) -> None: + self.selected_arcade = self.SelectedArcade() + + def set_machine(self, id: int, serial: Optional[str]) -> None: + self.selected_machine = self.SelectedMachine(id, serial) + + def clear_machine(self) -> None: + self.selected_machine = self.SelectedMachine() + + def set_last_err(self, err: str) -> None: + self.last_err = err + + def clear_last_err(self) -> None: + self.last_err = "" + + def clear_search_results(self) -> None: + self.search_results = [] + +state = State() +data: Data = None +loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() + +class MainView(Frame): + def __init__(self, screen: Screen): + super(MainView, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="ARTEMiS TUI" + ) + + layout = Layout([100], True) + self.add_layout(layout) + layout.add_widget(Button("User Management", self._user_mgmt)) + layout.add_widget(Button("Card Management", self._card_mgmt)) + layout.add_widget(Button("Arcade Management", self._arcade_mgmt)) + layout.add_widget(Button("Machine Management", self._mech_mgmt)) + layout.add_widget(Button("Quit", self._quit)) + + self.fix() + + def _user_mgmt(self): + self.save() + raise NextScene("User Management") + + def _card_mgmt(self): + self.save() + raise NextScene("Card Management") + + def _arcade_mgmt(self): + self.save() + raise NextScene("Arcade Management") + + def _mech_mgmt(self): + self.save() + raise NextScene("Mech Management") + + @staticmethod + def _quit(): + raise StopApplication("User pressed quit") + +class ManageUser(Frame): + def __init__(self, screen: Screen): + super(ManageUser, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="User Management" + ) + + layout = Layout([3]) + self.add_layout(layout) + layout.add_widget(Button("Create User", self._create_user)) + layout.add_widget(Button("Lookup User", self._lookup)) + layout.add_widget(Button("Edit User", self._edit_user, disabled=state.selected_user.id != 0)) + layout.add_widget(Button("Delete User", self._del_user, disabled=state.selected_user.id != 0)) + + usr_cards = [] + #if state.selected_user.id != 0: + #cards = data.card.get_user_cards(state.selected_user.id) + #for card in cards: + #usr_cards.append(card._asdict()) + + layout3 = Layout([100], True) + self.add_layout(layout3) + if len(usr_cards) > 0: + layout.add_widget(Divider()) + layout3.add_widget(RadioButtons( + [(f"{card['id']}\t{card['access_code']}\t{card['status']}", card['id']) for card in usr_cards], + "Cards:", + "usr_cards" + )) + layout3.add_widget(Divider()) + + layout2 = Layout([1, 1, 1, 1]) + self.add_layout(layout2) + a = Text("", f"status", readonly=True, disabled=True) + a.value = f"Selected User: {state.selected_user}" + layout2.add_widget(a) + layout2.add_widget(Button("Back", self._back), 3) + + self.fix() + + def _create_user(self): + self.save() + raise NextScene("Create User") + + def _lookup(self): + self.save() + raise NextScene("Lookup User") + + def _edit_user(self): + self.save() + raise NextScene("Lookup User") + + def _del_user(self): + self.save() + raise NextScene("Lookup User") + + def _back(self): + self.save() + raise NextScene("Main") + +class CreateUserView(Frame): + def __init__(self, screen: Screen): + super(CreateUserView, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="Create User" + ) + + layout = Layout([100], fill_frame=True) + self.add_layout(layout) + layout.add_widget(Text("Username:", "username")) + layout.add_widget(Text("Email:", "email")) + layout.add_widget(Text("Password:", "passwd")) + layout.add_widget(CheckBox("", "Add Card:", "is_add_card", )) + layout.add_widget(RadioButtons([ + ("User", "1"), + ("User Manager", "2"), + ("Arcde Manager", "4"), + ("Sysadmin", "8"), + ("Owner", "255"), + ], "Role:", "role")) + + layout3 = Layout([100]) + self.add_layout(layout3) + layout3.add_widget(Text("", f"status", readonly=True, disabled=True)) + + layout2 = Layout([1, 1, 1, 1]) + self.add_layout(layout2) + layout2.add_widget(Button("OK", self._ok), 0) + layout2.add_widget(Button("Cancel", self._cancel), 3) + + self.fix() + + def _ok(self): + self.save() + if not self.data.get("username"): + state.set_last_err("Username cannot be blank") + self.find_widget('status').value = state.last_err + self.screen.reset() + return + + state.clear_last_err() + self.find_widget('status').value = state.last_err + + if not self.data.get("passwd"): + pw = "".join( + secrets.choice(string.ascii_letters + string.digits) for i in range(20) + ) + else: + pw = self.data.get("passwd") + + hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt()) + + loop.run_until_complete(self._create_user_async(self.data.get("username"), hash.decode(), self.data.get("email"), self.data.get('role'))) + + raise NextScene("User Management") + + async def _create_user_async(self, username: str, password: str, email: Optional[str], role: str): + usr_id = await data.user.create_user( + username=username, + email=email if email else None, + password=password, + permission=int(role) + ) + + state.set_user(usr_id, username) + + def _cancel(self): + state.clear_last_err() + self.find_widget('status').value = state.last_err + raise NextScene("User Management") + +class SearchResultsView(Frame): + def __init__(self, screen: Screen): + super(CreateUserView, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="Search Results" + ) + + layout = Layout([100], fill_frame=True) + self.add_layout(layout) + layout.add_widget(Text("Username:", "username")) + + self.fix() + + def _ok(self): + self.save() + if not self.data.get("username"): + state.set_last_err("Username cannot be blank") + self.find_widget('status').value = state.last_err + self.screen.reset() + return + + state.clear_last_err() + self.find_widget('status').value = state.last_err + + if not self.data.get("passwd"): + pw = "".join( + secrets.choice(string.ascii_letters + string.digits) for i in range(20) + ) + else: + pw = self.data.get("passwd") + + hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt()) + + loop.run_until_complete(self._create_user_async(self.data.get("username"), hash.decode(), self.data.get("email"), self.data.get('role'))) + + raise NextScene("User Management") + + async def _create_user_async(self, username: str, password: str, email: Optional[str], role: str): + usr_id = await data.user.create_user( + username=username, + email=email if email else None, + password=password, + permission=int(role) + ) + + state.set_user(usr_id, username) + + def _cancel(self): + state.clear_last_err() + self.find_widget('status').value = state.last_err + raise NextScene("User Management") + +class LookupUserView(Frame): + def __init__(self, screen): + super(LookupUserView, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="Lookup User" + ) + + layout = Layout([1, 1], fill_frame=True) + self.add_layout(layout) + layout.add_widget(RadioButtons([ + ("Username", "1"), + ("Email", "2"), + ("Access Code", "3"), + ("User ID", "4"), + ], "Search By:", "search_type")) + layout.add_widget(Text("Search:", "search_str"), 1) + + layout3 = Layout([100]) + self.add_layout(layout3) + layout3.add_widget(Text("", f"status", readonly=True, disabled=True)) + + layout2 = Layout([1, 1, 1, 1]) + self.add_layout(layout2) + layout2.add_widget(Button("Search", self._lookup), 0) + layout2.add_widget(Button("Cancel", self._cancel), 3) + + self.fix() + + def _lookup(self): + self.save() + if not self.data.get("search_str"): + state.set_last_err("Search cannot be blank") + self.find_widget('status').value = state.last_err + self.screen.reset() + return + + state.clear_last_err() + self.find_widget('status').value = state.last_err + + search_type = self.data.get("search_type") + if search_type == "1": + loop.run_until_complete(self._lookup_user_by_username(self.data.get("search_str"))) + elif search_type == "2": + loop.run_until_complete(self._lookup_user_by_email(self.data.get("search_str"))) + elif search_type == "3": + loop.run_until_complete(self._lookup_user_by_access_code(self.data.get("search_str"))) + elif search_type == "4": + loop.run_until_complete(self._lookup_user_by_id(self.data.get("search_str"))) + else: + state.set_last_err("Unknown search type") + self.find_widget('status').value = state.last_err + self.screen.reset() + return + + if len(state.search_results) < 1: + state.set_last_err("Search returned no results") + self.find_widget('status').value = state.last_err + self.screen.reset() + return + + state.search_type = "user" + raise NextScene("Search Results") + + async def _lookup_user_by_id(self, user_id: str): + usr = await data.user.get_user(user_id) + + if usr is not None: + state.search_results = [usr] + + async def _lookup_user_by_username(self, username: str): + usr = await data.user.find_user_by_username(username) + + if usr is not None: + state.search_results = usr + + async def _lookup_user_by_email(self, email: str): + usr = await data.user.find_user_by_email(email) + + if usr is not None: + state.search_results = usr + + async def _lookup_user_by_access_code(self, access_code: str): + card = await data.card.get_card_by_access_code(access_code) + + if card is not None: + usr = await data.user.get_user(card['user']) + if usr is not None: + state.search_results = [usr] + + def _cancel(self): + state.clear_last_err() + self.find_widget('status').value = state.last_err + raise NextScene("User Management") + +def demo(screen:Screen, scene: Scene): + scenes = [ + Scene([MainView(screen)], -1, name="Main"), + Scene([ManageUser(screen)], -1, name="User Management"), + Scene([CreateUserView(screen)], -1, name="Create User"), + Scene([LookupUserView(screen)], -1, name="Lookup User"), + Scene([SearchResultsView(screen)], -1, name="Search Results"), + ] + + screen.play(scenes, stop_on_resize=False, start_scene=scene, allow_int=True) + +last_scene = None + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Database utilities") + parser.add_argument( + "--config", "-c", type=str, help="Config folder to use", default="config" + ) + args = parser.parse_args() + + cfg = CoreConfig() + if path.exists(f"{args.config}/core.yaml"): + cfg_dict = yaml.safe_load(open(f"{args.config}/core.yaml")) + cfg_dict.get("database", {})["loglevel"] = "info" + cfg.update(cfg_dict) + + if not path.exists(cfg.server.log_dir): + mkdir(cfg.server.log_dir) + + if not access(cfg.server.log_dir, W_OK): + print( + f"Log directory {cfg.server.log_dir} NOT writable, please check permissions" + ) + exit(1) + + data = Data(cfg) + + while True: + try: + Screen.wrapper(demo, catch_interrupt=True, arguments=[last_scene]) + exit(0) + except ResizeScreenError as e: + last_scene = e.scene From e8cd6e95966507f0d45b846c4d26ba968b04fc66 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Thu, 19 Dec 2024 00:17:00 -0500 Subject: [PATCH 071/130] tui: add user lookup --- tui.py | 102 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/tui.py b/tui.py index 06c51e9..9705c97 100644 --- a/tui.py +++ b/tui.py @@ -13,7 +13,7 @@ from core.data import Data from core.config import CoreConfig try: - from asciimatics.widgets import Frame, Layout, Text, Button, RadioButtons, CheckBox, Divider + from asciimatics.widgets import Frame, Layout, Text, Button, RadioButtons, CheckBox, Divider, Label from asciimatics.scene import Scene from asciimatics.screen import Screen from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication @@ -27,10 +27,10 @@ class State: def __init__(self, id: Optional[int] = None, name: Optional[str] = None): self.id = id self.name = name - + def __str__(self): if self.id is not None: - return f"{self.name} ({self.id})" if self.name else f"User {self.id}" + return f"{self.name} ({self.id})" if self.name else f"User{self.id:04d}" return "None" def __int__(self): @@ -171,39 +171,46 @@ class ManageUser(Frame): screen.width * 2 // 3, hover_focus=True, can_scroll=False, - title="User Management" + title="User Management", + on_load=self._redraw ) layout = Layout([3]) self.add_layout(layout) layout.add_widget(Button("Create User", self._create_user)) layout.add_widget(Button("Lookup User", self._lookup)) - layout.add_widget(Button("Edit User", self._edit_user, disabled=state.selected_user.id != 0)) - layout.add_widget(Button("Delete User", self._del_user, disabled=state.selected_user.id != 0)) - + + def _redraw(self): + self._layouts = [self._layouts[0]] + + layout = Layout([3]) + self.add_layout(layout) + layout.add_widget(Button("Edit User", self._edit_user, disabled=state.selected_user.id == 0 or state.selected_user.id is None)) + layout.add_widget(Button("Delete User", self._del_user, disabled=state.selected_user.id == 0 or state.selected_user.id is None)) + layout.add_widget((Divider())) + usr_cards = [] #if state.selected_user.id != 0: #cards = data.card.get_user_cards(state.selected_user.id) #for card in cards: #usr_cards.append(card._asdict()) - layout3 = Layout([100], True) - self.add_layout(layout3) if len(usr_cards) > 0: - layout.add_widget(Divider()) + layout3 = Layout([100], True) + self.add_layout(layout3) layout3.add_widget(RadioButtons( [(f"{card['id']}\t{card['access_code']}\t{card['status']}", card['id']) for card in usr_cards], "Cards:", "usr_cards" )) - layout3.add_widget(Divider()) + layout3.add_widget(Divider()) - layout2 = Layout([1, 1, 1, 1]) + layout2 = Layout([1, 1, 1]) self.add_layout(layout2) a = Text("", f"status", readonly=True, disabled=True) a.value = f"Selected User: {state.selected_user}" layout2.add_widget(a) - layout2.add_widget(Button("Back", self._back), 3) + layout2.add_widget(Button("Back", self._back), 2) self.fix() @@ -217,11 +224,11 @@ class ManageUser(Frame): def _edit_user(self): self.save() - raise NextScene("Lookup User") + raise NextScene("Edit User") def _del_user(self): self.save() - raise NextScene("Lookup User") + raise NextScene("Delete User") def _back(self): self.save() @@ -304,58 +311,51 @@ class CreateUserView(Frame): class SearchResultsView(Frame): def __init__(self, screen: Screen): - super(CreateUserView, self).__init__( + super(SearchResultsView, self).__init__( screen, screen.height * 2 // 3, screen.width * 2 // 3, hover_focus=True, can_scroll=False, - title="Search Results" + title="Search Results", + on_load=self._redraw ) - + + layout2 = Layout([1, 1, 1, 1]) + self.add_layout(layout2) + layout2.add_widget(Button("Select", self._select_current), 2) + layout2.add_widget(Button("Cancel", self._cancel), 2) + + def _redraw(self): + self._layouts = [self._layouts[0]] layout = Layout([100], fill_frame=True) self.add_layout(layout) - layout.add_widget(Text("Username:", "username")) + opts = [] + if state.search_type == "user": + layout.add_widget(Label(" ID | Username | Role | Email")) + layout.add_widget(Divider()) + + for usr in state.search_results: + name = str(usr['username']) + if len(name) < 8: + name = str(usr['username']) + ' ' * (8 - len(name)) + elif len(name) > 8: + name = usr['username'][:5] + "..." + + opts.append((f"{usr['id']:05d} | {name} | {usr['permissions']:08b} | {usr['email']}", state.SelectedUser(usr["id"], str(usr['username'])))) + + layout.add_widget(RadioButtons(opts, "", "selopt")) self.fix() - def _ok(self): + def _select_current(self): self.save() - if not self.data.get("username"): - state.set_last_err("Username cannot be blank") - self.find_widget('status').value = state.last_err - self.screen.reset() - return - - state.clear_last_err() - self.find_widget('status').value = state.last_err - - if not self.data.get("passwd"): - pw = "".join( - secrets.choice(string.ascii_letters + string.digits) for i in range(20) - ) - else: - pw = self.data.get("passwd") - - hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt()) - - loop.run_until_complete(self._create_user_async(self.data.get("username"), hash.decode(), self.data.get("email"), self.data.get('role'))) - + a = self.data.get('selopt') + state.set_user(a.id, a.name) raise NextScene("User Management") - - async def _create_user_async(self, username: str, password: str, email: Optional[str], role: str): - usr_id = await data.user.create_user( - username=username, - email=email if email else None, - password=password, - permission=int(role) - ) - - state.set_user(usr_id, username) def _cancel(self): state.clear_last_err() - self.find_widget('status').value = state.last_err raise NextScene("User Management") class LookupUserView(Frame): From f8307649902bc76fb61fd1699c704408f33dccab Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Thu, 19 Dec 2024 00:21:39 -0500 Subject: [PATCH 072/130] tui: fix minor alignment issue --- tui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tui.py b/tui.py index 9705c97..39db651 100644 --- a/tui.py +++ b/tui.py @@ -332,7 +332,7 @@ class SearchResultsView(Frame): self.add_layout(layout) opts = [] if state.search_type == "user": - layout.add_widget(Label(" ID | Username | Role | Email")) + layout.add_widget(Label(" ID | Username | Role | Email")) layout.add_widget(Divider()) for usr in state.search_results: From 5475b52336f517446e2d3600902fdea3f1e0e1e1 Mon Sep 17 00:00:00 2001 From: beerpsi Date: Thu, 19 Dec 2024 13:02:08 +0700 Subject: [PATCH 073/130] [chunithm] support luminous+ --- docs/game_specific_info.md | 15 +- example_config/chuni.yaml | 3 + readme.md | 1 + titles/chuni/base.py | 24 +- titles/chuni/const.py | 7 +- titles/chuni/index.py | 20 +- titles/chuni/luminous.py | 477 +++++++++++++++++++++++++---------- titles/chuni/luminousplus.py | 170 +++++++++++++ titles/chuni/new.py | 2 + 9 files changed, 570 insertions(+), 149 deletions(-) create mode 100644 titles/chuni/luminousplus.py diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 638d993..16c15e7 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -60,13 +60,14 @@ Games listed below have been tested and confirmed working. ### SDHD/SDBT -| Version ID | Version Name | -| ---------- | ------------------- | -| 11 | CHUNITHM NEW!! | -| 12 | CHUNITHM NEW PLUS!! | -| 13 | CHUNITHM SUN | -| 14 | CHUNITHM SUN PLUS | -| 15 | CHUNITHM LUMINOUS | +| Version ID | Version Name | +| ---------- | ---------------------- | +| 11 | CHUNITHM NEW!! | +| 12 | CHUNITHM NEW PLUS!! | +| 13 | CHUNITHM SUN | +| 14 | CHUNITHM SUN PLUS | +| 15 | CHUNITHM LUMINOUS | +| 16 | CHUNITHM LUMINOUS PLUS | ### Importer diff --git a/example_config/chuni.yaml b/example_config/chuni.yaml index ce2f683..a3781d6 100644 --- a/example_config/chuni.yaml +++ b/example_config/chuni.yaml @@ -40,6 +40,9 @@ version: 15: rom: 2.20.00 data: 2.20.00 + 16: + rom: 2.25.00 + data: 2.25.00 crypto: encrypted_only: False diff --git a/readme.md b/readme.md index e863e52..e44c137 100644 --- a/readme.md +++ b/readme.md @@ -30,6 +30,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + SUN + SUN PLUS + LUMINOUS + + LUMINOUS PLUS + crossbeats REV. + Crossbeats REV. diff --git a/titles/chuni/base.py b/titles/chuni/base.py index 37bcbb3..5bec5bf 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -8,7 +8,7 @@ import pytz from core.config import CoreConfig from titles.chuni.config import ChuniConfig -from titles.chuni.const import ChuniConstants, ItemKind +from titles.chuni.const import ChuniConstants, FavoriteItemKind, ItemKind from titles.chuni.database import ChuniData @@ -1014,6 +1014,28 @@ class ChuniBase: ) await self.data.profile.put_net_battle(user_id, net_battle) + # New in LUMINOUS PLUS + if "userFavoriteMusicList" in upsert: + # musicId, orderId + music_ids = set(int(m["musicId"]) for m in upsert["userFavoriteMusicList"]) + current_favorites = await self.data.item.get_all_favorites( + user_id, self.version, fav_kind=FavoriteItemKind.MUSIC + ) + + if current_favorites is None: + current_favorites = [] + + current_favorite_ids = set(x.favId for x in current_favorites) + keep_ids = current_favorite_ids.intersection(music_ids) + deleted_ids = current_favorite_ids - keep_ids + added_ids = music_ids - keep_ids + + for fav_id in deleted_ids: + await self.data.item.delete_favorite_music(user_id, self.version, fav_id) + + for fav_id in added_ids: + await self.data.item.put_favorite_music(user_id, self.version, fav_id) + return {"returnCode": "1"} async def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict: diff --git a/titles/chuni/const.py b/titles/chuni/const.py index 45fd498..d0d73d5 100644 --- a/titles/chuni/const.py +++ b/titles/chuni/const.py @@ -25,6 +25,7 @@ class ChuniConstants: VER_CHUNITHM_SUN = 13 VER_CHUNITHM_SUN_PLUS = 14 VER_CHUNITHM_LUMINOUS = 15 + VER_CHUNITHM_LUMINOUS_PLUS = 16 VERSION_NAMES = [ "CHUNITHM", @@ -43,6 +44,7 @@ class ChuniConstants: "CHUNITHM SUN", "CHUNITHM SUN PLUS", "CHUNITHM LUMINOUS", + "CHUNITHM LUMINOUS PLUS", ] SCORE_RANK_INTERVALS_OLD = [ @@ -98,6 +100,7 @@ class MapAreaConditionType(IntEnum): TROPHY_OBTAINED = 3 + RANK_SSSP = 18 RANK_SSS = 19 RANK_SSP = 20 RANK_SS = 21 @@ -127,7 +130,7 @@ class ItemKind(IntEnum): FRAME = 2 """ "Frame" is the background for the gauge/score/max combo display - shown during gameplay. This item cannot be equipped (as of LUMINOUS) + shown during gameplay. This item cannot be equipped (as of LUMINOUS PLUS) and is hardcoded to the current game's version. """ @@ -146,7 +149,7 @@ class ItemKind(IntEnum): ULTIMA_UNLOCK = 12 """This only applies to ULTIMA difficulties that are *not* unlocked by - SS-ing EXPERT+MASTER. + reaching S rank on EXPERT difficulty or above. """ diff --git a/titles/chuni/index.py b/titles/chuni/index.py index 144f770..080c041 100644 --- a/titles/chuni/index.py +++ b/titles/chuni/index.py @@ -36,6 +36,7 @@ from .newplus import ChuniNewPlus from .sun import ChuniSun from .sunplus import ChuniSunPlus from .luminous import ChuniLuminous +from .luminousplus import ChuniLuminousPlus class ChuniServlet(BaseServlet): def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None: @@ -64,6 +65,7 @@ class ChuniServlet(BaseServlet): ChuniSun, ChuniSunPlus, ChuniLuminous, + ChuniLuminousPlus, ] self.logger = logging.getLogger("chuni") @@ -107,6 +109,7 @@ class ChuniServlet(BaseServlet): f"{ChuniConstants.VER_CHUNITHM_SUN_PLUS}_int": 36, ChuniConstants.VER_CHUNITHM_LUMINOUS: 8, f"{ChuniConstants.VER_CHUNITHM_LUMINOUS}_int": 8, + ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS: 56, } for version, keys in self.game_cfg.crypto.keys.items(): @@ -235,8 +238,10 @@ class ChuniServlet(BaseServlet): internal_ver = ChuniConstants.VER_CHUNITHM_SUN elif version >= 215 and version < 220: # SUN PLUS internal_ver = ChuniConstants.VER_CHUNITHM_SUN_PLUS - elif version >= 220: # LUMINOUS + elif version >= 220 and version < 225: # LUMINOUS internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS + elif version >= 225: # LUMINOUS PLUS + internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS elif game_code == "SDGS": # Int if version < 105: # SUPERSTAR internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS @@ -250,8 +255,10 @@ class ChuniServlet(BaseServlet): internal_ver = ChuniConstants.VER_CHUNITHM_SUN elif version >= 125 and version < 130: # SUN PLUS internal_ver = ChuniConstants.VER_CHUNITHM_SUN_PLUS - elif version >= 130: # LUMINOUS + elif version >= 130 and version < 135: # LUMINOUS internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS + elif version >= 135: # LUMINOUS PLUS + internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32: # If we get a 32 character long hex string, it's a hash and we're @@ -311,8 +318,10 @@ class ChuniServlet(BaseServlet): return Response(zlib.compress(b'{"stat": "0"}')) try: - unzip = zlib.decompress(req_raw) - + if request.headers.get("x-debug") is not None: + unzip = req_raw + else: + unzip = zlib.decompress(req_raw) except zlib.error as e: self.logger.error( f"Failed to decompress v{version} {endpoint} request -> {e}" @@ -352,6 +361,9 @@ class ChuniServlet(BaseServlet): self.logger.debug(f"Response {resp}") + if request.headers.get("x-debug") is not None: + return Response(json.dumps(resp, ensure_ascii=False).encode("utf-8")) + zipped = zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8")) if not encrtped: diff --git a/titles/chuni/luminous.py b/titles/chuni/luminous.py index 8f02820..6fcc9ea 100644 --- a/titles/chuni/luminous.py +++ b/titles/chuni/luminous.py @@ -2,9 +2,13 @@ from datetime import timedelta from typing import Dict from core.config import CoreConfig -from titles.chuni.sunplus import ChuniSunPlus -from titles.chuni.const import ChuniConstants, MapAreaConditionLogicalOperator, MapAreaConditionType from titles.chuni.config import ChuniConfig +from titles.chuni.const import ( + ChuniConstants, + MapAreaConditionLogicalOperator, + MapAreaConditionType, +) +from titles.chuni.sunplus import ChuniSunPlus class ChuniLuminous(ChuniSunPlus): @@ -18,7 +22,7 @@ class ChuniLuminous(ChuniSunPlus): # Does CARD MAKER 1.35 work this far up? user_data["lastDataVersion"] = "2.20.00" return user_data - + async def handle_get_user_c_mission_api_request(self, data: Dict) -> Dict: user_id = data["userId"] mission_id = data["missionId"] @@ -28,7 +32,7 @@ class ChuniLuminous(ChuniSunPlus): mission_data = await self.data.item.get_cmission(user_id, mission_id) progress_data = await self.data.item.get_cmission_progress(user_id, mission_id) - + if mission_data and progress_data: point = mission_data["point"] @@ -48,12 +52,14 @@ class ChuniLuminous(ChuniSunPlus): "userCMissionProgressList": progress_list, } - async def handle_get_user_net_battle_ranking_info_api_request(self, data: Dict) -> Dict: + async def handle_get_user_net_battle_ranking_info_api_request( + self, data: Dict + ) -> Dict: user_id = data["userId"] net_battle = {} net_battle_data = await self.data.profile.get_net_battle(user_id) - + if net_battle_data: net_battle = { "isRankUpChallengeFailed": net_battle_data["isRankUpChallengeFailed"], @@ -94,131 +100,135 @@ class ChuniLuminous(ChuniSunPlus): # (event ID 14214) was imported into ARTEMiS, we disable the requirement # for this trophy. if 14214 in event_by_id: - mission_in_progress_end_date = (event_by_id[14214]["startDate"] - timedelta(hours=2)).strftime(self.date_time_format) - - conditions.extend([ - { - "mapAreaId": 2206201, # BlythE ULTIMA - "length": 1, - # Obtain the trophy "MISSION in progress". - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6832, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": mission_in_progress_end_date, - } - ], - }, - { - "mapAreaId": 2206202, # PRIVATE SERVICE ULTIMA - "length": 1, - # Obtain the trophy "MISSION in progress". - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6832, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": mission_in_progress_end_date, - } - ], - }, - { - "mapAreaId": 2206203, # New York Back Raise - "length": 1, - # SS NightTheater's EXPERT chart and get the title - # "今宵、劇場に映し出される景色とは――――。" - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6833, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - }, - ], - }, - { - "mapAreaId": 2206204, # Spasmodic - "length": 2, - # - Get 1 miss on Random (any difficulty) and get the title "当たり待ち" - # - Get 1 miss on 花たちに希望を (any difficulty) and get the title "花たちに希望を" - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6834, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - }, - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6835, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - }, - ], - }, - { - "mapAreaId": 2206205, # ΩΩPARTS - "length": 2, - # - S Sage EXPERT to get the title "ﾏﾀｰﾘ進行ｷﾎﾞﾝﾇ" - # - Equip this title and play cab-to-cab with another person with this title - # to get "ﾏﾀｰﾘしようよ". Disabled because it is difficult to play cab2cab - # on data setups. A network operator may consider re-enabling it by uncommenting - # the second condition. - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6836, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - }, - # { - # "type": MapAreaConditionType.TROPHY_OBTAINED.value, - # "conditionId": 6837, - # "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - # "startDate": start_date, - # "endDate": "2099-12-31 00:00:00.0", - # }, - ], - }, - { - "mapAreaId": 2206206, # Blow My Mind - "length": 1, - # SS on CHAOS EXPERT, Hydra EXPERT, Surive EXPERT and Jakarta PROGRESSION EXPERT - # to get the title "Can you hear me?" - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.TROPHY_OBTAINED.value, - "conditionId": 6838, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - }, - ], - }, - { - "mapAreaId": 2206207, # VALLIS-NERIA - "length": 6, - # Finish the 6 other areas - "mapAreaConditionList": [ - { - "type": MapAreaConditionType.MAP_AREA_CLEARED.value, - "conditionId": x, - "logicalOpe": MapAreaConditionLogicalOperator.AND.value, - "startDate": start_date, - "endDate": "2099-12-31 00:00:00.0", - } - for x in range(2206201, 2206207) - ], - }, - ]) - + mission_in_progress_end_date = ( + event_by_id[14214]["startDate"] - timedelta(hours=2) + ).strftime(self.date_time_format) + + conditions.extend( + [ + { + "mapAreaId": 2206201, # BlythE ULTIMA + "length": 1, + # Obtain the trophy "MISSION in progress". + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6832, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": mission_in_progress_end_date, + } + ], + }, + { + "mapAreaId": 2206202, # PRIVATE SERVICE ULTIMA + "length": 1, + # Obtain the trophy "MISSION in progress". + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6832, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": mission_in_progress_end_date, + } + ], + }, + { + "mapAreaId": 2206203, # New York Back Raise + "length": 1, + # SS NightTheater's EXPERT chart and get the title + # "今宵、劇場に映し出される景色とは――――。" + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6833, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + ], + }, + { + "mapAreaId": 2206204, # Spasmodic + "length": 2, + # - Get 1 miss on Random (any difficulty) and get the title "当たり待ち" + # - Get 1 miss on 花たちに希望を (any difficulty) and get the title "花たちに希望を" + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6834, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6835, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + ], + }, + { + "mapAreaId": 2206205, # ΩΩPARTS + "length": 2, + # - S Sage EXPERT to get the title "ﾏﾀｰﾘ進行ｷﾎﾞﾝﾇ" + # - Equip this title and play cab-to-cab with another person with this title + # to get "ﾏﾀｰﾘしようよ". Disabled because it is difficult to play cab2cab + # on data setups. A network operator may consider re-enabling it by uncommenting + # the second condition. + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6836, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + # { + # "type": MapAreaConditionType.TROPHY_OBTAINED.value, + # "conditionId": 6837, + # "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + # "startDate": start_date, + # "endDate": "2099-12-31 00:00:00.0", + # }, + ], + }, + { + "mapAreaId": 2206206, # Blow My Mind + "length": 1, + # SS on CHAOS EXPERT, Hydra EXPERT, Surive EXPERT and Jakarta PROGRESSION EXPERT + # to get the title "Can you hear me?" + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": 6838, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + ], + }, + { + "mapAreaId": 2206207, # VALLIS-NERIA + "length": 6, + # Finish the 6 other areas + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.MAP_AREA_CLEARED.value, + "conditionId": x, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + } + for x in range(2206201, 2206207) + ], + }, + ] + ) + # LUMINOUS ep. I if 14005 in event_by_id: start_date = event_by_id[14005]["startDate"].strftime(self.date_time_format) @@ -226,7 +236,7 @@ class ChuniLuminous(ChuniSunPlus): if not mystic_area_1_added: conditions.append(mystic_area_1_conditions) mystic_area_1_added = True - + mystic_area_1_conditions["length"] += 1 mystic_area_1_conditions["mapAreaConditionList"].append( { @@ -254,15 +264,15 @@ class ChuniLuminous(ChuniSunPlus): ], } ) - + # LUMINOUS ep. II if 14251 in event_by_id: start_date = event_by_id[14251]["startDate"].strftime(self.date_time_format) - + if not mystic_area_1_added: conditions.append(mystic_area_1_conditions) mystic_area_1_added = True - + mystic_area_1_conditions["length"] += 1 mystic_area_1_conditions["mapAreaConditionList"].append( { @@ -291,6 +301,203 @@ class ChuniLuminous(ChuniSunPlus): } ) + # LUMINOUS ep. III + if 14481 in event_by_id: + start_date = event_by_id[14481]["startDate"].strftime(self.date_time_format) + + if not mystic_area_1_added: + conditions.append(mystic_area_1_conditions) + mystic_area_1_added = True + + mystic_area_1_conditions["length"] += 1 + mystic_area_1_conditions["mapAreaConditionList"].append( + { + "type": MapAreaConditionType.MAP_CLEARED.value, + "conditionId": 3020703, + "logicalOpe": MapAreaConditionLogicalOperator.OR.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + } + ) + + conditions.append( + { + "mapAreaId": 3229304, # Mystic Rainbow of LUMINOUS Area 4, + "length": 1, + # Unlocks when LUMINOUS ep. III is completed. + "mapAreaConditionList": [ + { + "type": MapAreaConditionType.MAP_CLEARED.value, + "conditionId": 3020703, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date, + "endDate": "2099-12-31 00:00:00.0", + }, + ], + } + ) + + # 1UM1N0U5 ep. 111 + if 14483 in event_by_id: + start_date = event_by_id[14483]["startDate"].replace( + hour=0, minute=0, second=0 + ) + + # conditions to unlock the 6 "Key of ..." area in the map + # for the first 14 days: Defandour MASTER AJ, crazy (about you) MASTER AJ, Halcyon ULTIMA SSS + title_conditions = [ + { + "type": MapAreaConditionType.ALL_JUSTICE.value, + "conditionId": 258103, # Defandour MASTER + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": ( + start_date + timedelta(days=14) - timedelta(seconds=1) + ).strftime(self.date_time_format), + }, + { + "type": MapAreaConditionType.ALL_JUSTICE.value, + "conditionId": 258003, # crazy (about you) MASTER + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": ( + start_date + timedelta(days=14) - timedelta(seconds=1) + ).strftime(self.date_time_format), + }, + { + "type": MapAreaConditionType.RANK_SSS.value, + "conditionId": 17304, # Halcyon ULTIMA + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": ( + start_date + timedelta(days=14) - timedelta(seconds=1) + ).strftime(self.date_time_format), + }, + ] + + # For each next 14 days, the conditions are lowered to SS+, S+, S, and then always unlocked + for i, typ in enumerate( + [ + MapAreaConditionType.RANK_SSP.value, + MapAreaConditionType.RANK_SP.value, + MapAreaConditionType.RANK_S.value, + MapAreaConditionType.ALWAYS_UNLOCKED.value, + ] + ): + start = (start_date + timedelta(days=14 * (i + 1))).strftime( + self.date_time_format + ) + + if typ != MapAreaConditionType.ALWAYS_UNLOCKED.value: + end = ( + start_date + timedelta(days=14 * (i + 2)) - timedelta(seconds=1) + ).strftime(self.date_time_format) + + title_conditions.extend( + [ + { + "type": typ, + "conditionId": condition_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start, + "endDate": end, + } + for condition_id in {17304, 258003, 258103} + ] + ) + else: + end = "2099-12-31 00:00:00" + + title_conditions.append( + { + "type": typ, + "conditionId": 0, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start, + "endDate": end, + } + ) + + # actually add all the conditions + for map_area_id in range(3229201, 3229207): + conditions.append( + { + "mapAreaId": map_area_id, + "length": len(title_conditions), + "mapAreaConditionList": title_conditions, + } + ) + + # Ultimate Force + # For the first 14 days, the condition is to obtain all 9 "Key of ..." titles + # Afterwards, the condition is the 6 "Key of ..." titles that you can obtain + # by playing the 6 areas, as well as obtaining specific ranks on + # [CRYSTAL_ACCESS] / Strange Love / βlαnoir + ultimate_force_conditions = [] + + # Trophies obtained by playing the 6 areas + for trophy_id in {6851, 6853, 6855, 6857, 6858, 6860}: + ultimate_force_conditions.append( + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": trophy_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": "2099-12-31 00:00:00", + } + ) + + # βlαnoir MASTER SSS+ / Strange Love MASTER SSS+ / [CRYSTAL_ACCESS] MASTER SSS+ + for trophy_id in {6852, 6854, 6856}: + ultimate_force_conditions.append( + { + "type": MapAreaConditionType.TROPHY_OBTAINED.value, + "conditionId": trophy_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": ( + start_date + timedelta(days=14) - timedelta(seconds=1) + ).strftime(self.date_time_format), + } + ) + + # For each next 14 days, the rank conditions for the 3 songs lowers + # Finally, the Ultimate Force area is unlocked as soon as you finish the 6 other areas. + for i, typ in enumerate( + [ + MapAreaConditionType.RANK_SSS.value, + MapAreaConditionType.RANK_SS.value, + MapAreaConditionType.RANK_S.value, + ] + ): + start = (start_date + timedelta(days=14 * (i + 1))).strftime( + self.date_time_format + ) + + end = ( + start_date + timedelta(days=14 * (i + 2)) - timedelta(seconds=1) + ).strftime(self.date_time_format) + + ultimate_force_conditions.extend( + [ + { + "type": typ, + "conditionId": condition_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start, + "endDate": end, + } + for condition_id in {109403, 212103, 244203} + ] + ) + + conditions.append( + { + "mapAreaId": 3229207, + "length": len(ultimate_force_conditions), + "mapAreaConditionList": ultimate_force_conditions, + } + ) return { "length": len(conditions), diff --git a/titles/chuni/luminousplus.py b/titles/chuni/luminousplus.py new file mode 100644 index 0000000..659b39d --- /dev/null +++ b/titles/chuni/luminousplus.py @@ -0,0 +1,170 @@ +from datetime import timedelta +from typing import Dict + +from core.config import CoreConfig +from titles.chuni.config import ChuniConfig +from titles.chuni.const import ChuniConstants, MapAreaConditionLogicalOperator, MapAreaConditionType +from titles.chuni.luminous import ChuniLuminous + + +class ChuniLuminousPlus(ChuniLuminous): + def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None: + super().__init__(core_cfg, game_cfg) + self.version = ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS + + async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: + user_data = await super().handle_cm_get_user_preview_api_request(data) + + # Does CARD MAKER 1.35 work this far up? + user_data["lastDataVersion"] = "2.25.00" + return user_data + + async def handle_get_user_c_mission_list_api_request(self, data: Dict) -> Dict: + user_id = int(data["userId"]) + user_mission_list_request = data["userCMissionList"] + + user_mission_list = [] + + for request in user_mission_list_request: + user_id = int(request["userId"]) + mission_id = int(request["missionId"]) + point = int(request["point"]) + + mission_data = await self.data.item.get_cmission(user_id, mission_id) + progress_data = await self.data.item.get_cmission_progress(user_id, mission_id) + + if mission_data is None or progress_data is None: + continue + + point = mission_data.point + user_mission_progress_list = [ + { + "order": progress.order, + "stage": progress.stage, + "progress": progress.progress, + } + for progress in progress_data + ] + + user_mission_list.append( + { + "userId": user_id, + "missionId": mission_id, + "point": point, + "userCMissionProgressList": user_mission_progress_list, + }, + ) + + return { + "userId": user_id, + "userCMissionList": user_mission_list, + } + + async def handle_get_game_map_area_condition_api_request(self, data: Dict) -> Dict: + # There is no game data for this, everything is server side. + # However, we can selectively show/hide events as data is imported into the server. + events = await self.data.static.get_enabled_events(self.version) + event_by_id = {evt["eventId"]: evt for evt in events} + conditions = [] + + # LUMINOUS ep. Ascension + if ep_ascension := event_by_id.get(15512): + start_date = ep_ascension["startDate"].replace(hour=0, minute=0, second=0) + + # Finish LUMINOUS ep. VII to unlock LUMINOUS ep. Ascension. + task_track_map_conditions = [ + { + "type": MapAreaConditionType.MAP_CLEARED.value, + "conditionId": 3020707, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start_date.strftime(self.date_time_format), + "endDate": "2099-12-31 00:00:00", + } + ] + + # You also need to reach a specific rank on Acid God MASTER. + # This condition lowers every 7 days. + # After the first 4 weeks, you only need to finish ep. VII. + for i, typ in enumerate([ + MapAreaConditionType.RANK_SSSP.value, + MapAreaConditionType.RANK_SSS.value, + MapAreaConditionType.RANK_SS.value, + MapAreaConditionType.RANK_S.value, + ]): + start = start_date + timedelta(days=7 * i) + end = start_date + timedelta(days=7 * (i + 1)) - timedelta(seconds=1) + + task_track_map_conditions.append( + { + "type": typ, + "conditionId": 265103, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start.strftime(self.date_time_format), + "endDate": end.strftime(self.date_time_format), + } + ) + + conditions.extend( + [ + { + "mapAreaId": map_area_id, + "length": len(task_track_map_conditions), + "mapAreaConditionList": task_track_map_conditions, + } + for map_area_id in {3220801, 3220802, 3220803, 3220804} + ] + ) + + # To unlock the final map area (Forsaken Tale), achieve a specific rank + # on the 4 task tracks in the previous map areas. This condition also lowers + # every 7 days, similar to Acid God. + # After 28 days, you only need to finish the other 4 areas in ep. Ascension. + forsaken_tale_conditions = [] + + for i, typ in enumerate([ + MapAreaConditionType.RANK_SSSP.value, + MapAreaConditionType.RANK_SSS.value, + MapAreaConditionType.RANK_SS.value, + MapAreaConditionType.RANK_S.value, + ]): + start = start_date + timedelta(days=7 * i) + end = start_date + timedelta(days=7 * (i + 1)) - timedelta(seconds=1) + + forsaken_tale_conditions.extend( + [ + { + "type": typ, + "conditionId": condition_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": start.strftime(self.date_time_format), + "endDate": end.strftime(self.date_time_format), + } + for condition_id in {98203, 108603, 247503, 233903} + ] + ) + + forsaken_tale_conditions.extend( + [ + { + "type": MapAreaConditionType.MAP_AREA_CLEARED.value, + "conditionId": map_area_id, + "logicalOpe": MapAreaConditionLogicalOperator.AND.value, + "startDate": (start_date + timedelta(days=28)).strftime(self.date_time_format), + "endDate": "2099-12-31 00:00:00", + } + for map_area_id in {3220801, 3220802, 3220803, 3220804} + ] + ) + + conditions.append( + { + "mapAreaId": 3220805, + "length": len(forsaken_tale_conditions), + "mapAreaConditionList": forsaken_tale_conditions, + } + ) + + return { + "length": len(conditions), + "gameMapAreaConditionList": conditions, + } diff --git a/titles/chuni/new.py b/titles/chuni/new.py index 15d2b6c..a3aa1a3 100644 --- a/titles/chuni/new.py +++ b/titles/chuni/new.py @@ -36,6 +36,8 @@ class ChuniNew(ChuniBase): return "215" if self.version == ChuniConstants.VER_CHUNITHM_LUMINOUS: return "220" + if self.version == ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS: + return "225" async def handle_get_game_setting_api_request(self, data: Dict) -> Dict: # use UTC time and convert it to JST time by adding +9 From b81d5c9cc5e2e59db39bf1c4c4fa047779ff3154 Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Thu, 19 Dec 2024 01:37:50 -0500 Subject: [PATCH 074/130] adb: fix minor logging typo --- core/aimedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/aimedb.py b/core/aimedb.py index 6d5bd57..79fd9a3 100644 --- a/core/aimedb.py +++ b/core/aimedb.py @@ -270,7 +270,7 @@ class AimedbServlette(): ac = card['access_code'] self.logger.info( - f"idm {idm} ipm {req.pmm.zfill(16)} -> access_code {ac}" + f"idm {idm} pmm {req.pmm.zfill(16)} -> access_code {ac}" ) return ADBFelicaLookupResponse.from_req(req.head, ac) From 0cf41ff389511e67a3f6e1805f43c3976a44979c Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Fri, 20 Dec 2024 17:40:55 -0500 Subject: [PATCH 075/130] TUI: add card management screen --- tui.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/tui.py b/tui.py index 39db651..b140964 100644 --- a/tui.py +++ b/tui.py @@ -190,19 +190,29 @@ class ManageUser(Frame): layout.add_widget((Divider())) usr_cards = [] - #if state.selected_user.id != 0: - #cards = data.card.get_user_cards(state.selected_user.id) - #for card in cards: - #usr_cards.append(card._asdict()) + if state.selected_user.id != 0: + cards = loop.run_until_complete(data.card.get_user_cards(state.selected_user.id)) + for card in cards: + usr_cards.append(card._asdict()) if len(usr_cards) > 0: layout3 = Layout([100], True) self.add_layout(layout3) + + card_status = "Available" + if card['is_locked'] and card['is_banned']: + card_status = "Locked and Banned" + if card['is_locked']: + card_status = "Locked" + if card['is_banned']: + card_status = "Banned" + layout3.add_widget(RadioButtons( - [(f"{card['id']}\t{card['access_code']}\t{card['status']}", card['id']) for card in usr_cards], + [(f"{card['id']} | {card['access_code']} | {card_status} | {card['memo'] if card['memo'] else '-'}", state.SelectedCard(card['id'], card['access_code'])) for card in usr_cards], "Cards:", "usr_cards" )) + layout3.add_widget(Button('Select Card', self._sel_card)) layout3.add_widget(Divider()) layout2 = Layout([1, 1, 1]) @@ -214,6 +224,12 @@ class ManageUser(Frame): self.fix() + def _sel_card(self): + self.save() + a = self.data.get('usr_cards') + state.set_card(a.id, a.access_code) + raise NextScene("Card Management") + def _create_user(self): self.save() raise NextScene("Create User") @@ -234,6 +250,62 @@ class ManageUser(Frame): self.save() raise NextScene("Main") +class ManageCard(Frame): + def __init__(self, screen: Screen): + super(ManageCard, self).__init__( + screen, + screen.height * 2 // 3, + screen.width * 2 // 3, + hover_focus=True, + can_scroll=False, + title="Card Management", + on_load=self._redraw + ) + + layout = Layout([3]) + self.add_layout(layout) + layout.add_widget(Button("Create Card", self._create_card)) + layout.add_widget(Button("Lookup Card", self._lookup)) + + def _redraw(self): + self._layouts = [self._layouts[0]] + + layout = Layout([3]) + self.add_layout(layout) + layout.add_widget(Button("Edit Card", self._edit_card, disabled=state.selected_card.id == 0 or state.selected_card.id is None)) + layout.add_widget(Button("Reassign Card", self._edit_card, disabled=state.selected_card.id == 0 or state.selected_card.id is None)) + layout.add_widget(Button("Delete Card", self._del_card, disabled=state.selected_card.id == 0 or state.selected_card.id is None)) + layout.add_widget((Divider())) + + layout2 = Layout([1, 1, 1]) + self.add_layout(layout2) + a = Text("", f"status", readonly=True, disabled=True) + a.value = f"Selected Card: {state.selected_card}" + layout2.add_widget(a) + layout2.add_widget(Button("Back", self._back), 2) + + self.fix() + + def _create_card(self): + self.save() + raise NextScene("Create Card") + + def _lookup(self): + self.save() + raise NextScene("Lookup Card") + + def _edit_card(self): + self.save() + raise NextScene("Edit Card") + + def _del_card(self): + self.save() + raise NextScene("Delete Card") + + def _back(self): + self.save() + raise NextScene("Main") + class CreateUserView(Frame): def __init__(self, screen: Screen): super(CreateUserView, self).__init__( @@ -463,6 +535,7 @@ def demo(screen:Screen, scene: Scene): Scene([CreateUserView(screen)], -1, name="Create User"), Scene([LookupUserView(screen)], -1, name="Lookup User"), Scene([SearchResultsView(screen)], -1, name="Search Results"), + Scene([ManageCard(screen)], -1, name="Card Management"), ] screen.play(scenes, stop_on_resize=False, start_scene=scene, allow_int=True) From ab64eea5c9311caae73302e2261e32ec4fb55f01 Mon Sep 17 00:00:00 2001 From: akanyan Date: Mon, 30 Dec 2024 18:31:22 +0000 Subject: [PATCH 076/130] ongeki: read music version from the xml --- titles/ongeki/read.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index a804956..98aaf67 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -27,6 +27,27 @@ class OngekiReader(BaseReader): self.logger.error(f"Invalid ongeki version {version}") exit(1) + def parse_version(self, troot) -> int: + version_ids = { + "1000": OngekiConstants.VER_ONGEKI, + "1005": OngekiConstants.VER_ONGEKI_PLUS, + "1010": OngekiConstants.VER_ONGEKI_SUMMER, + "1015": OngekiConstants.VER_ONGEKI_SUMMER_PLUS, + "1020": OngekiConstants.VER_ONGEKI_RED, + "1025": OngekiConstants.VER_ONGEKI_RED_PLUS, + "1030": OngekiConstants.VER_ONGEKI_BRIGHT, + "1035": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, + "1040": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY + } + + node = troot.find("VersionID").find("id") + + if node.text not in version_ids: + self.logger.warn(f"Unknown VersionID {node.text}") + return OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY + + return version_ids[node.text] + async def read(self) -> None: data_dirs = [] if self.bin_dir is not None: @@ -44,17 +65,6 @@ class OngekiReader(BaseReader): async def read_card(self, base_dir: str) -> None: self.logger.info(f"Reading cards from {base_dir}...") - version_ids = { - "1000": OngekiConstants.VER_ONGEKI, - "1005": OngekiConstants.VER_ONGEKI_PLUS, - "1010": OngekiConstants.VER_ONGEKI_SUMMER, - "1015": OngekiConstants.VER_ONGEKI_SUMMER_PLUS, - "1020": OngekiConstants.VER_ONGEKI_RED, - "1025": OngekiConstants.VER_ONGEKI_RED_PLUS, - "1030": OngekiConstants.VER_ONGEKI_BRIGHT, - "1035": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, - } - for root, dirs, files in os.walk(base_dir): for dir in dirs: if os.path.exists(f"{root}/{dir}/Card.xml"): @@ -92,11 +102,10 @@ class OngekiReader(BaseReader): troot.find("ChoKaikaSkillID").find("id").text ) - version = version_ids[troot.find("VersionID").find("id").text] card_number = troot.find("CardNumberString").text await self.data.static.put_card( - version, + self.parse_version(troot), card_id, name=name, charaId=chara_id, @@ -151,6 +160,7 @@ class OngekiReader(BaseReader): title = name.find("str").text artist = troot.find("ArtistName").find("str").text genre = troot.find("Genre").find("str").text + version = self.parse_version(troot) fumens = troot.find("FumenData") for fumens_data in fumens.findall("FumenData"): @@ -164,7 +174,7 @@ class OngekiReader(BaseReader): ) await self.data.static.put_chart( - self.version, song_id, chart_id, title, artist, genre, level + version, song_id, chart_id, title, artist, genre, level ) self.logger.info(f"Added song {song_id} chart {chart_id}") From fa667d15f2116f52b2cf2c3e1f103b31ed630b12 Mon Sep 17 00:00:00 2001 From: akanyan Date: Mon, 6 Jan 2025 18:39:49 +0000 Subject: [PATCH 077/130] ongeki: proper handling of music ranking list --- ...remove_ongeki_static_music_ranking_list.py | 40 +++++++++++++++++++ titles/ongeki/base.py | 32 +++++++++------ titles/ongeki/schema/score.py | 16 +++++++- titles/ongeki/schema/static.py | 19 --------- 4 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 core/data/alembic/versions/9c42e54a27fe_remove_ongeki_static_music_ranking_list.py diff --git a/core/data/alembic/versions/9c42e54a27fe_remove_ongeki_static_music_ranking_list.py b/core/data/alembic/versions/9c42e54a27fe_remove_ongeki_static_music_ranking_list.py new file mode 100644 index 0000000..31d8526 --- /dev/null +++ b/core/data/alembic/versions/9c42e54a27fe_remove_ongeki_static_music_ranking_list.py @@ -0,0 +1,40 @@ +"""remove ongeki_static_music_ranking_list + +Revision ID: 9c42e54a27fe +Revises: 41f77ef50588 +Create Date: 2025-01-06 18:24:16.306748 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '9c42e54a27fe' +down_revision = '41f77ef50588' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ongeki_static_music_ranking_uk', table_name='ongeki_static_music_ranking_list') + op.drop_table('ongeki_static_music_ranking_list') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('ongeki_static_music_ranking_list', + sa.Column('id', mysql.INTEGER(), autoincrement=True, nullable=False), + sa.Column('version', mysql.INTEGER(), autoincrement=False, nullable=False), + sa.Column('musicId', mysql.INTEGER(), autoincrement=False, nullable=False), + sa.Column('point', mysql.INTEGER(), autoincrement=False, nullable=False), + sa.Column('userName', mysql.VARCHAR(length=255), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_collate='utf8mb4_0900_ai_ci', + mysql_default_charset='utf8mb4', + mysql_engine='InnoDB' + ) + op.create_index('ongeki_static_music_ranking_uk', 'ongeki_static_music_ranking_list', ['version', 'musicId'], unique=True) + # ### end Alembic commands ### diff --git a/titles/ongeki/base.py b/titles/ongeki/base.py index 1bebb4d..bebbe4c 100644 --- a/titles/ongeki/base.py +++ b/titles/ongeki/base.py @@ -157,19 +157,27 @@ class OngekiBase: return {"type": data["type"], "length": 0, "gameIdlistList": []} async def handle_get_game_ranking_api_request(self, data: Dict) -> Dict: - game_ranking_list = await self.data.static.get_ranking_list(self.version) - - ranking_list = [] - for music in game_ranking_list: - tmp = music._asdict() - ranking_list.append(tmp) + try: + date = datetime.now(pytz.timezone('Asia/Tokyo')) - timedelta(days=1,hours=7) - if ranking_list is None: - return {"length": 0, "gameRankingList": []} - return { - "type": data["type"], - "gameRankingList": ranking_list, - } + # type 1 - current ranking; type 2 - previous ranking + if data["type"] == 2: + date = date - timedelta(1) + + rankings = await self.data.score.get_rankings(date) + + if not rankings or (data["type"] == 1 and len(rankings) < 10): + return {"type": data["type"], "gameRankingList": []} + + ranking_list = [] + for count, music_id in rankings: + ranking_list.append({"id": music_id, "point": count, "userName": ""}) + + return {"type": data["type"], "gameRankingList": ranking_list} + + except Exception as e: + self.logger.error(f"Error while getting game ranking: {e}") + return {"type": data["type"], "gameRankingList": []} async def handle_get_game_point_api_request(self, data: Dict) -> Dict: get_game_point = await self.data.static.get_static_game_point() diff --git a/titles/ongeki/schema/score.py b/titles/ongeki/schema/score.py index 178cf29..f10d676 100644 --- a/titles/ongeki/schema/score.py +++ b/titles/ongeki/schema/score.py @@ -4,11 +4,13 @@ from sqlalchemy import Column, Table, UniqueConstraint from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row from sqlalchemy.schema import ForeignKey -from sqlalchemy.sql import select +from sqlalchemy.sql import select, func from sqlalchemy.types import TIMESTAMP, Boolean, Float, Integer, String from core.data.schema import BaseData, metadata +from datetime import datetime, timedelta + score_best: Table = Table( "ongeki_score_best", metadata, @@ -209,6 +211,18 @@ class OngekiScoreData(BaseData): return None return result.lastrowid + async def get_rankings(self, date: datetime) -> Optional[List[Row]]: + sql = ( + select([func.count(playlog.c.id), playlog.c.musicId]) + .where(playlog.c.playDate == date.date()) + .group_by(playlog.c.musicId) + .order_by(func.count(playlog.c.id).desc()) + .limit(10) + ) + result = await self.execute(sql) + if result: + return result.fetchall() + async def put_playlog(self, aime_id: int, playlog_data: Dict) -> Optional[int]: playlog_data["user"] = aime_id diff --git a/titles/ongeki/schema/static.py b/titles/ongeki/schema/static.py index 85a9df4..30b2767 100644 --- a/titles/ongeki/schema/static.py +++ b/titles/ongeki/schema/static.py @@ -98,18 +98,6 @@ cards = Table( mysql_charset="utf8mb4", ) -music_ranking = Table( - "ongeki_static_music_ranking_list", - metadata, - Column("id", Integer, primary_key=True, nullable=False), - Column("version", Integer, nullable=False), - Column("musicId", Integer, nullable=False), - Column("point", Integer, nullable=False), - Column("userName", String(255)), - UniqueConstraint("version", "musicId", name="ongeki_static_music_ranking_uk"), - mysql_charset="utf8mb4", -) - rewards = Table( "ongeki_static_rewards", metadata, @@ -425,13 +413,6 @@ class OngekiStaticData(BaseData): return None return result.fetchone() - async def get_ranking_list(self, version: int) -> Optional[List[Dict]]: - sql = select(music_ranking.c.musicId.label('id'), music_ranking.c.point, music_ranking.c.userName).where(music_ranking.c.version == version) - result = await self.execute(sql) - if result is None: - return None - return result.fetchall() - async def put_reward(self, version: int, rewardId: int, rewardname: str, itemKind: int, itemId: int) -> Optional[int]: sql = insert(rewards).values( version=version, From 59a3c28134c913317dd4683c63deaae58a72e36d Mon Sep 17 00:00:00 2001 From: akanyan Date: Mon, 20 Jan 2025 22:34:05 +0000 Subject: [PATCH 078/130] ongeki: use the latest applicable version --- titles/ongeki/schema/profile.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/titles/ongeki/schema/profile.py b/titles/ongeki/schema/profile.py index b42a0a3..2196c9d 100644 --- a/titles/ongeki/schema/profile.py +++ b/titles/ongeki/schema/profile.py @@ -277,8 +277,8 @@ class OngekiProfileData(BaseData): async def get_profile_name(self, aime_id: int, version: int) -> Optional[str]: sql = select(profile.c.userName).where( - and_(profile.c.user == aime_id, profile.c.version == version) - ) + and_(profile.c.user == aime_id, profile.c.version <= version) + ).order_by(profile.c.version.desc()) result = await self.execute(sql) if result is None: @@ -294,7 +294,8 @@ class OngekiProfileData(BaseData): sql = ( select([profile, option]) .join(option, profile.c.user == option.c.user) - .filter(and_(profile.c.user == aime_id, profile.c.version == version)) + .filter(and_(profile.c.user == aime_id, profile.c.version <= version)) + .order_by(profile.c.version.desc()) ) result = await self.execute(sql) @@ -306,9 +307,9 @@ class OngekiProfileData(BaseData): sql = select(profile).where( and_( profile.c.user == aime_id, - profile.c.version == version, + profile.c.version <= version, ) - ) + ).order_by(profile.c.version.desc()) result = await self.execute(sql) if result is None: From f3f0569755f6ef741488cc15446c9e43c685c5a3 Mon Sep 17 00:00:00 2001 From: Galexion Date: Mon, 27 Jan 2025 19:50:49 +0000 Subject: [PATCH 079/130] Fixes Capitalization on CrossBeats Read.py --- titles/cxb/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/cxb/read.py b/titles/cxb/read.py index 9a2ae98..c8d4616 100644 --- a/titles/cxb/read.py +++ b/titles/cxb/read.py @@ -38,7 +38,7 @@ class CxbReader(BaseReader): self.logger.info(f"Read csv from {bin_dir}") try: - fullPath = bin_dir + "/export.csv" + fullPath = bin_dir + "/Export.csv" with open(fullPath, encoding="UTF-8") as fp: reader = csv.DictReader(fp) for row in reader: From 0f52b89033f43f861a4ff81dda264839959400ba Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Fri, 21 Feb 2025 23:51:59 -0500 Subject: [PATCH 080/130] remove deprecated warn --- core/aimedb.py | 14 ++++++------ core/allnet.py | 4 ++-- core/data/database.py | 4 ++-- core/data/schema/card.py | 2 +- core/frontend.py | 6 +++--- core/mucha.py | 4 ++-- core/title.py | 4 ++-- titles/chuni/frontend.py | 2 +- titles/chuni/schema/profile.py | 8 +++---- titles/cxb/index.py | 6 +++--- titles/cxb/read.py | 2 +- titles/diva/frontend.py | 4 ++-- titles/idac/schema/item.py | 30 +++++++++++++------------- titles/idac/schema/profile.py | 12 +++++------ titles/mai2/base.py | 2 +- titles/mai2/frontend.py | 2 +- titles/ongeki/read.py | 2 +- titles/ongeki/schema/profile.py | 2 +- titles/pokken/base.py | 6 +++--- titles/sao/base.py | 38 ++++++++++++++++----------------- titles/sao/index.py | 2 +- titles/sao/read.py | 2 +- 22 files changed, 79 insertions(+), 79 deletions(-) diff --git a/core/aimedb.py b/core/aimedb.py index b99d697..ea2ff4a 100644 --- a/core/aimedb.py +++ b/core/aimedb.py @@ -137,7 +137,7 @@ class AimedbServlette(): resp_bytes = resp elif resp is None: # Nothing to send, probably a goodbye - self.logger.warn(f"None return by handler for {name}") + self.logger.warning(f"None return by handler for {name}") return else: @@ -177,7 +177,7 @@ class AimedbServlette(): async def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse: req = ADBLookupRequest(data) if req.access_code == "00000000000000000000": - self.logger.warn(f"All-zero access code from {req.head.keychip_id}") + self.logger.warning(f"All-zero access code from {req.head.keychip_id}") ret = ADBLookupResponse.from_req(req.head, -1) ret.head.status = ADBStatus.BAN_SYS return ret @@ -208,7 +208,7 @@ class AimedbServlette(): async def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse: req = ADBLookupRequest(data) if req.access_code == "00000000000000000000": - self.logger.warn(f"All-zero access code from {req.head.keychip_id}") + self.logger.warning(f"All-zero access code from {req.head.keychip_id}") ret = ADBLookupExResponse.from_req(req.head, -1) ret.head.status = ADBStatus.BAN_SYS return ret @@ -254,7 +254,7 @@ class AimedbServlette(): req = ADBFelicaLookupRequest(data) idm = req.idm.zfill(16) if idm == "0000000000000000": - self.logger.warn(f"All-zero IDm from {req.head.keychip_id}") + self.logger.warning(f"All-zero IDm from {req.head.keychip_id}") ret = ADBFelicaLookupResponse.from_req(req.head, "00000000000000000000") ret.head.status = ADBStatus.BAN_SYS return ret @@ -283,7 +283,7 @@ class AimedbServlette(): idm = req.idm.zfill(16) if idm == "0000000000000000": - self.logger.warn(f"All-zero IDm from {req.head.keychip_id}") + self.logger.warning(f"All-zero IDm from {req.head.keychip_id}") ret = ADBFelicaLookupResponse.from_req(req.head, "00000000000000000000") ret.head.status = ADBStatus.BAN_SYS return ret @@ -323,7 +323,7 @@ class AimedbServlette(): idm = req.idm.zfill(16) if idm == "0000000000000000": - self.logger.warn(f"All-zero IDm from {req.head.keychip_id}") + self.logger.warning(f"All-zero IDm from {req.head.keychip_id}") ret = ADBFelicaLookupExResponse.from_req(req.head, -1, "00000000000000000000") ret.head.status = ADBStatus.BAN_SYS return ret @@ -382,7 +382,7 @@ class AimedbServlette(): user_id = -1 if req.access_code == "00000000000000000000": - self.logger.warn(f"All-zero access code from {req.head.keychip_id}") + self.logger.warning(f"All-zero access code from {req.head.keychip_id}") ret = ADBLookupResponse.from_req(req.head, -1) ret.head.status = ADBStatus.BAN_SYS return ret diff --git a/core/allnet.py b/core/allnet.py index 43dde1f..0b6006b 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -609,7 +609,7 @@ class BillingServlet: traces.append(tmp) except KeyError as e: - self.logger.warn(f"Tracelog failed to parse: {e}") + self.logger.warning(f"Tracelog failed to parse: {e}") kc_serial_bytes = req.keychipid.encode() @@ -648,7 +648,7 @@ class BillingServlet: ) if req.traceleft > 0: - self.logger.warn(f"{req.traceleft} unsent tracelogs") + self.logger.warning(f"{req.traceleft} unsent tracelogs") kc_playlimit = req.playlimit kc_nearfull = req.nearfull diff --git a/core/data/database.py b/core/data/database.py index fb36ebd..f98fe33 100644 --- a/core/data/database.py +++ b/core/data/database.py @@ -223,7 +223,7 @@ class Data: async def legacy_upgrade(self) -> bool: vers = await self.base.execute("SELECT * FROM schema_versions") if vers is None: - self.logger.warn("Cannot legacy upgrade, schema_versions table unavailable!") + self.logger.warning("Cannot legacy upgrade, schema_versions table unavailable!") return False db_vers = {} @@ -252,7 +252,7 @@ class Data: game_codes = getattr(mod, "game_codes", []) for game in game_codes: if game not in db_vers: - self.logger.warn(f"{game} does not have an antry in schema_versions, skipping") + self.logger.warning(f"{game} does not have an antry in schema_versions, skipping") continue now_ver = int(db_vers[game]) + 1 diff --git a/core/data/schema/card.py b/core/data/schema/card.py index 254b19e..8705820 100644 --- a/core/data/schema/card.py +++ b/core/data/schema/card.py @@ -123,7 +123,7 @@ class CardData(BaseData): result = await self.execute(sql) if result is None: - self.logger.warn(f"Failed to update last login time for {access_code}") + self.logger.warning(f"Failed to update last login time for {access_code}") async def get_card_by_idm(self, idm: str) -> Optional[Row]: result = await self.execute(aime_card.select(aime_card.c.idm == idm)) diff --git a/core/frontend.py b/core/frontend.py index c593828..2e2fc2d 100644 --- a/core/frontend.py +++ b/core/frontend.py @@ -435,7 +435,7 @@ class FE_User(FE_Base): if user_id: if not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD) and user_id != usr_sesh.user_id: - self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view user {user_id}") + self.logger.warning(f"User {usr_sesh.user_id} does not have permission to view user {user_id}") return RedirectResponse("/user/", 303) else: @@ -939,7 +939,7 @@ class FE_Arcade(FE_Base): usr_sesh = self.validate_session(request) if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD): - self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!") + self.logger.warning(f"User {usr_sesh.user_id} does not have permission to view shops!") return RedirectResponse("/gate/", 303) if not shop_id: @@ -983,7 +983,7 @@ class FE_Machine(FE_Base): usr_sesh = self.validate_session(request) if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD): - self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!") + self.logger.warning(f"User {usr_sesh.user_id} does not have permission to view shops!") return RedirectResponse("/gate/", 303) if not cab_id: diff --git a/core/mucha.py b/core/mucha.py index 22e4789..d715a19 100644 --- a/core/mucha.py +++ b/core/mucha.py @@ -64,7 +64,7 @@ class MuchaServlet: self.logger.debug(f"Mucha request {vars(req)}") if not req.gameCd or not req.gameVer or not req.sendDate or not req.countryCd or not req.serialNum: - self.logger.warn(f"Missing required fields - {vars(req)}") + self.logger.warning(f"Missing required fields - {vars(req)}") return PlainTextResponse("RESULTS=000") minfo = self.mucha_registry.get(req.gameCd, {}) @@ -133,7 +133,7 @@ class MuchaServlet: self.logger.info(f"Allow unknown serial {netid} ({sn_decrypt}) to auth") else: - self.logger.warn(f'Auth failed for NetID {netid}') + self.logger.warning(f'Auth failed for NetID {netid}') return PlainTextResponse("RESULTS=000") self.logger.debug(f"Mucha response {vars(resp)}") diff --git a/core/title.py b/core/title.py index 016e09a..9165628 100644 --- a/core/title.py +++ b/core/title.py @@ -86,11 +86,11 @@ class BaseServlet: return (False, [], []) async def render_POST(self, request: Request) -> bytes: - self.logger.warn(f"Game Does not dispatch POST") + self.logger.warning(f"Game Does not dispatch POST") return Response() async def render_GET(self, request: Request) -> bytes: - self.logger.warn(f"Game Does not dispatch GET") + self.logger.warning(f"Game Does not dispatch GET") return Response() class TitleServlet: diff --git a/titles/chuni/frontend.py b/titles/chuni/frontend.py index 1faa23b..876e873 100644 --- a/titles/chuni/frontend.py +++ b/titles/chuni/frontend.py @@ -715,7 +715,7 @@ class ChuniFrontend(FE_Base): elif o < 0x7F and o > 0x20: new_name_full += chr(o + 0xFEE0) elif o <= 0x7F: - self.logger.warn(f"Invalid ascii character {o:02X}") + self.logger.warning(f"Invalid ascii character {o:02X}") return RedirectResponse("/gate/?e=4", 303) else: new_name_full += x diff --git a/titles/chuni/schema/profile.py b/titles/chuni/schema/profile.py index 8d71ba6..1362c9f 100644 --- a/titles/chuni/schema/profile.py +++ b/titles/chuni/schema/profile.py @@ -765,7 +765,7 @@ class ChuniProfileData(BaseData): existing_team = self.get_team_by_id(team_id) if existing_team is None or "userTeamPoint" not in existing_team: - self.logger.warn( + self.logger.warning( f"update_team: Failed to update team! team id: {team_id}. Existing team data not found." ) return False @@ -795,7 +795,7 @@ class ChuniProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"update_team: Failed to update team! team id: {team_id}" ) return False @@ -813,7 +813,7 @@ class ChuniProfileData(BaseData): playcount_sql = await self.execute(select(profile.c.playCount)) if playcount_sql is None: - self.logger.warn( + self.logger.warning( f"get_overview: Couldn't pull playcounts" ) return 0 @@ -842,7 +842,7 @@ class ChuniProfileData(BaseData): result = await self.execute(sql) if result is None: - self.logger.warn( + self.logger.warning( f"put_profile_rating: Could not insert {rating_type}, aime_id: {aime_id}", ) return diff --git a/titles/cxb/index.py b/titles/cxb/index.py index a4e7085..e97c8c3 100644 --- a/titles/cxb/index.py +++ b/titles/cxb/index.py @@ -180,7 +180,7 @@ class CxbServlet(BaseServlet): internal_ver = CxbConstants.VER_CROSSBEATS_REV_SUNRISE_S2 if not hasattr(self.versions[internal_ver], func_to_find): - self.logger.warn(f"{version_string} has no handler for filetype {filetype} / {func_to_find}") + self.logger.warning(f"{version_string} has no handler for filetype {filetype} / {func_to_find}") return JSONResponse({"data":""}) self.logger.info(f"{version_string} request for filetype {filetype}") @@ -209,7 +209,7 @@ class CxbServlet(BaseServlet): func_to_find = f"handle_action_{subcmd}_request" if not hasattr(self.versions[0], func_to_find): - self.logger.warn(f"No handler for action {subcmd} request") + self.logger.warning(f"No handler for action {subcmd} request") return Response() self.logger.info(f"Action {subcmd} Request") @@ -238,7 +238,7 @@ class CxbServlet(BaseServlet): func_to_find = f"handle_auth_{subcmd}_request" if not hasattr(self.versions[0], func_to_find): - self.logger.warn(f"No handler for auth {subcmd} request") + self.logger.warning(f"No handler for auth {subcmd} request") return Response() self.logger.info(f"Action {subcmd} Request") diff --git a/titles/cxb/read.py b/titles/cxb/read.py index c8d4616..73a95d1 100644 --- a/titles/cxb/read.py +++ b/titles/cxb/read.py @@ -32,7 +32,7 @@ class CxbReader(BaseReader): await self.read_csv(self.bin_dir) else: - self.logger.warn(f"{self.bin_dir} does not exist, nothing to import") + self.logger.warning(f"{self.bin_dir} does not exist, nothing to import") async def read_csv(self, bin_dir: str) -> None: self.logger.info(f"Read csv from {bin_dir}") diff --git a/titles/diva/frontend.py b/titles/diva/frontend.py index 0b50dc4..30ee6ac 100644 --- a/titles/diva/frontend.py +++ b/titles/diva/frontend.py @@ -130,7 +130,7 @@ class DivaFrontend(FE_Base): elif o < 0x7F and o > 0x20: new_name_full += chr(o + 0xFEE0) elif o <= 0x7F: - self.logger.warn(f"Invalid ascii character {o:02X}") + self.logger.warning(f"Invalid ascii character {o:02X}") return RedirectResponse("/gate/?e=4", 303) else: new_name_full += x @@ -167,7 +167,7 @@ class DivaFrontend(FE_Base): elif o < 0x7F and o > 0x20: new_lv_full += chr(o + 0xFEE0) elif o <= 0x7F: - self.logger.warn(f"Invalid ascii character {o:02X}") + self.logger.warning(f"Invalid ascii character {o:02X}") return RedirectResponse("/gate/?e=4", 303) else: new_lv_full += x diff --git a/titles/idac/schema/item.py b/titles/idac/schema/item.py index d617cd9..c3596a4 100644 --- a/titles/idac/schema/item.py +++ b/titles/idac/schema/item.py @@ -772,7 +772,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_car: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_car: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -784,7 +784,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_ticket: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_ticket: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -796,7 +796,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_story: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_story: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -811,7 +811,7 @@ class IDACItemData(BaseData): result = await self.execute(sql) if result is None: - self.logger.warn( + self.logger.warning( f"put_story_episode_play_status: Failed to update! aime_id: {aime_id}" ) return None @@ -828,7 +828,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_story_episode: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_story_episode: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -843,7 +843,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_story_episode_difficulty: Failed to update! aime_id: {aime_id}" ) return None @@ -857,7 +857,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_course: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_course: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -872,7 +872,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_time_trial: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_time_trial: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -884,7 +884,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_challenge: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_challenge: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -898,7 +898,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_theory_course: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_theory_course: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -912,7 +912,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_theory_partner: Failed to update! aime_id: {aime_id}" ) return None @@ -928,7 +928,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_theory_running: Failed to update! aime_id: {aime_id}" ) return None @@ -942,7 +942,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_vs_info: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_vs_info: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -956,7 +956,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"putstamp: Failed to update! aime_id: {aime_id}" ) return None @@ -976,7 +976,7 @@ class IDACItemData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_timetrial_event: Failed to update! aime_id: {aime_id}" ) return None diff --git a/titles/idac/schema/profile.py b/titles/idac/schema/profile.py index bb6593b..6eb2254 100644 --- a/titles/idac/schema/profile.py +++ b/titles/idac/schema/profile.py @@ -360,7 +360,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_profile: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_profile: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -372,7 +372,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_profile_config: Failed to update! aime_id: {aime_id}" ) return None @@ -386,7 +386,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_profile_avatar: Failed to update! aime_id: {aime_id}" ) return None @@ -403,7 +403,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_profile_rank: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_profile_rank: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -418,7 +418,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn(f"put_profile_stock: Failed to update! aime_id: {aime_id}") + self.logger.warning(f"put_profile_stock: Failed to update! aime_id: {aime_id}") return None return result.lastrowid @@ -433,7 +433,7 @@ class IDACProfileData(BaseData): result = await self.execute(conflict) if result is None: - self.logger.warn( + self.logger.warning( f"put_profile_theory: Failed to update! aime_id: {aime_id}" ) return None diff --git a/titles/mai2/base.py b/titles/mai2/base.py index e148533..5d1c767 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -514,7 +514,7 @@ class Mai2Base: continue # present period hasn't started yet, move onto the next one if (present['endDate'] and present['endDate'].timestamp() < datetime.now().timestamp()): - self.logger.warn(f"Present {present['id']} ended on {present['endDate']} and should be removed") + self.logger.warning(f"Present {present['id']} ended on {present['endDate']} and should be removed") continue # present period ended, move onto the next one test = await self.data.item.get_item(data["userId"], present['itemKind'], present['itemId']) diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index 9667eca..f8a7176 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -212,7 +212,7 @@ class Mai2Frontend(FE_Base): elif o < 0x7F and o > 0x20: new_name_full += chr(o + 0xFEE0) elif o <= 0x7F: - self.logger.warn(f"Invalid ascii character {o:02X}") + self.logger.warning(f"Invalid ascii character {o:02X}") return RedirectResponse("/gate/?e=4", 303) else: new_name_full += x diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index 98aaf67..18f1e8a 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -43,7 +43,7 @@ class OngekiReader(BaseReader): node = troot.find("VersionID").find("id") if node.text not in version_ids: - self.logger.warn(f"Unknown VersionID {node.text}") + self.logger.warning(f"Unknown VersionID {node.text}") return OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY return version_ids[node.text] diff --git a/titles/ongeki/schema/profile.py b/titles/ongeki/schema/profile.py index b42a0a3..db6a323 100644 --- a/titles/ongeki/schema/profile.py +++ b/titles/ongeki/schema/profile.py @@ -554,7 +554,7 @@ class OngekiProfileData(BaseData): result = await self.execute(sql) if result is None: - self.logger.warn( + self.logger.warning( f"put_profile_rating_{rating_type}: Could not insert rating entries, aime_id: {aime_id}", ) return diff --git a/titles/pokken/base.py b/titles/pokken/base.py index 78ca863..f9a622a 100644 --- a/titles/pokken/base.py +++ b/titles/pokken/base.py @@ -38,7 +38,7 @@ class PokkenBase: pcbid = request.register_pcb.pcb_id if not pcbid.isdigit() or len(pcbid) != 12 or \ not pcbid.startswith(f"{PokkenConstants.SERIAL_IDENT[0]}{PokkenConstants.SERIAL_REGIONS[0]}{PokkenConstants.SERIAL_ROLES[0]}{PokkenConstants.SERIAL_CAB_IDENTS[0]}"): - self.logger.warn(f"Bad PCBID {pcbid}") + self.logger.warning(f"Bad PCBID {pcbid}") res.result = 0 return res @@ -49,12 +49,12 @@ class PokkenBase: minfo = await self.data.arcade.get_machine(netid) if not minfo and not self.core_cfg.server.allow_unregistered_serials: - self.logger.warn(f"netID {netid} does not belong to any shop!") + self.logger.warning(f"netID {netid} does not belong to any shop!") res.result = 0 return res elif not minfo: - self.logger.warn(f"Orphaned netID {netid} allowed to connect") + self.logger.warning(f"Orphaned netID {netid} allowed to connect") locid = 0 else: diff --git a/titles/sao/base.py b/titles/sao/base.py index 759d35b..45cee9f 100644 --- a/titles/sao/base.py +++ b/titles/sao/base.py @@ -132,7 +132,7 @@ class SaoBase: await self.data.profile.add_yui_medals(user_id, medal_num) else: - self.logger.warn(f"User {user_id} Unhandled reward type {reward_type} -> {reward}") + self.logger.warning(f"User {user_id} Unhandled reward type {reward_type} -> {reward}") async def hero_default_skills(self, skill_table_id: int) -> List[int]: skills = await self.data.static.get_skill_table_by_subid(skill_table_id) @@ -370,7 +370,7 @@ class SaoBase: if not card: # Validate that we're talking to a phone if not int(pmm[2:4], 16) in self.data.card.moble_os_codes: - self.logger.warn(f"{req.serial_no} looked up non-moble chip ID {cid}!") + self.logger.warning(f"{req.serial_no} looked up non-moble chip ID {cid}!") return SaoGetAccessCodeByKeitaiResponse("").make() # TODO: Actual felica moble registration @@ -378,7 +378,7 @@ class SaoBase: #ac = await self.data.card.register_felica_moble_ac(idm, pmm) # if we didn't get an access code, fail hard if not ac: - self.logger.warn(f"Failed to register access code for chip ID {cid} requested by {req.serial_no}") + self.logger.warning(f"Failed to register access code for chip ID {cid} requested by {req.serial_no}") return SaoGetAccessCodeByKeitaiResponse("").make() self.logger.info(f"Successfully registered moble felica access code {ac} for chip ID {cid} requested by {req.serial_no}") @@ -416,14 +416,14 @@ class SaoBase: async def handle_c126(self, header: SaoRequestHeader, request: bytes, src_ip: str) -> bytes: # common/validation_error_notification req = SaoValidationErrorNotificationRequest(header, request) - self.logger.warn(f"User {req.user_id} on {'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + self.logger.warning(f"User {req.user_id} on {'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + f"Validation error: {req.send_protocol_name} || {req.send_data_to_fraud_value} || {req.send_data_to_modification_value}") return SaoNoopResponse(GameconnectCmd.VALIDATION_ERROR_NOTIFICATION_RESPONSE).make() async def handle_c128(self, header: SaoRequestHeader, request: bytes, src_ip: str) -> bytes: # common/power_cutting_return_notification req = SaoPowerCuttingReturnNotification(header, request) - self.logger.warn(f"User {req.user_id} on {'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + self.logger.warning(f"User {req.user_id} on {'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + f"Power outage return: Act Type {req.last_act_type} || {req.remaining_ticket_num} Remaining Tickets || {req.remaining_credit_num} Remaining Credits") return SaoNoopResponse(GameconnectCmd.POWER_CUTTING_RETURN_NOTIFICATION_RESPONSE).make() @@ -437,7 +437,7 @@ class SaoBase: async def handle_c12c(self, header: SaoRequestHeader, request: bytes, src_ip: str) -> bytes: # common/matching_error_notification req = SaoMatchingErrorNotificationRequest(header, request) - self.logger.warn(f"{'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + self.logger.warning(f"{'game' if req.cabinet_type == 0 else 'terminal'} {req.serial_no} @ {req.store_name} ({src_ip} | Place ID {req.place_id}) " \ + f"Matching error: {req.matching_error_data_list[0]}") return SaoNoopResponse(GameconnectCmd.MATCHING_ERROR_NOTIFICATION_RESPONSE).make() @@ -547,22 +547,22 @@ class SaoBase: card = await self.data.profile.get_hero_card(req.profile_card_code) if not card: - self.logger.warn(f"User {req.user_id} scanned unregistered QR code {req.profile_card_code}") + self.logger.warning(f"User {req.user_id} scanned unregistered QR code {req.profile_card_code}") return resp.make() hero = await self.data.item.get_hero_log_by_id(card['user_hero_id']) if not hero: # Shouldn't happen - self.logger.warn(f"User {req.user_id} scanned QR code {req.profile_card_code} but does not have hero entry {card['user_hero_id']}") + self.logger.warning(f"User {req.user_id} scanned QR code {req.profile_card_code} but does not have hero entry {card['user_hero_id']}") return resp.make() hero_static_data = await self.data.static.get_hero_by_id(hero['hero_log_id']) if not hero_static_data: # Shouldn't happen - self.logger.warn(f"No entry for hero {hero['hero_log_id']}, please run read.py") + self.logger.warning(f"No entry for hero {hero['hero_log_id']}, please run read.py") return resp.make() profile = await self.data.profile.get_profile(card['user']) if not profile: # Shouldn't happen - self.logger.warn(f"No profile for user {card['user']}, something broke") + self.logger.warning(f"No profile for user {card['user']}, something broke") return resp.make() self.logger.info(f"User {req.user_id} scanned QR code {req.profile_card_code}") @@ -587,7 +587,7 @@ class SaoBase: card = await self.data.profile.get_resource_card(req.resource_card_code) # TODO: use count if not card: - self.logger.warn(f"No resource card with serial {req.resource_card_code} exists!") + self.logger.warning(f"No resource card with serial {req.resource_card_code} exists!") resp.header.err_status = 4832 # Theres a few error codes but none seem to do anything? # Also not sure if it should be this or result @@ -603,7 +603,7 @@ class SaoBase: resp = SaoScanQrQuestResourceCardResponse(card['common_reward_type'], card['common_reward_id'], card['holographic_flag']) else: - self.logger.warn(f"No resource card with serial {req.resource_card_code} exists!") + self.logger.warning(f"No resource card with serial {req.resource_card_code} exists!") resp = SaoScanQrQuestResourceCardResponse() resp.header.err_status = 4832 # Theres a few error codes but none seem to do anything? # Also not sure if it should be this or result @@ -719,7 +719,7 @@ class SaoBase: append = HeroLogUserData.from_args(hero) hero_static = await self.data.static.get_hero_by_id(hero['hero_log_id']) if not hero_static: - self.logger.warn(f"No hero for id {hero['hero_log_id']}, please run reader") + self.logger.warning(f"No hero for id {hero['hero_log_id']}, please run reader") resp.hero_log_user_data_list.append(append) continue @@ -757,7 +757,7 @@ class SaoBase: e = EquipmentUserData.from_args(equipment) weapon_static = await self.data.static.get_equipment_by_id(equipment['equipment_id']) if not weapon_static: - self.logger.warn(f"No equipment for id {equipment['equipment_id']}, please run reader") + self.logger.warning(f"No equipment for id {equipment['equipment_id']}, please run reader") resp.equipment_user_data_list.append(e) continue @@ -917,7 +917,7 @@ class SaoBase: continue else: - self.logger.warn(f"Unhandled disposal type {disposal.common_reward_type}") + self.logger.warning(f"Unhandled disposal type {disposal.common_reward_type}") await self.data.profile.add_col(req.user_id, get_col) return SaoDisposalResourceResponse(get_col).make() @@ -978,7 +978,7 @@ class SaoBase: await self.data.item.remove_hero_log(x.user_common_reward_id) else: - self.logger.warn(f"Unhandled ype {x.common_reward_type}! (running {hero_exp})") + self.logger.warning(f"Unhandled ype {x.common_reward_type}! (running {hero_exp})") hero_exp = int(hero_exp * 1.5) await self.data.item.add_hero_xp(req.origin_user_hero_log_id, hero_exp) @@ -1072,7 +1072,7 @@ class SaoBase: await self.data.item.remove_hero_log(x.user_common_reward_id) else: - self.logger.warn(f"Unhandled ype {x.common_reward_type}! (running {equipment_exp})") + self.logger.warning(f"Unhandled ype {x.common_reward_type}! (running {equipment_exp})") equipment_exp = int(equipment_exp * 1.5) await self.data.item.add_equipment_enhancement_exp(req_data.origin_user_equipment_id, equipment_exp) @@ -1322,7 +1322,7 @@ class SaoBase: # TODO pass else: - self.logger.warn(f"Unhandled EX Bonus condition {condition}") + self.logger.warning(f"Unhandled EX Bonus condition {condition}") resp.play_end_response_data[0].ex_bonus_data_list.append(QuestScenePlayEndExBonusData.from_args(table_id, ach_thing)) @@ -1550,7 +1550,7 @@ class SaoBase: # TODO pass else: - self.logger.warn(f"Unhandled EX Bonus condition {condition}") + self.logger.warning(f"Unhandled EX Bonus condition {condition}") resp.play_end_response_data[0].ex_bonus_data_list.append(QuestScenePlayEndExBonusData.from_args(table_id, ach_thing)) diff --git a/titles/sao/index.py b/titles/sao/index.py index 8a775c6..43484d8 100644 --- a/titles/sao/index.py +++ b/titles/sao/index.py @@ -103,7 +103,7 @@ class SaoServlet(BaseServlet): req_raw = await request.body() if len(req_raw) < 40: - self.logger.warn(f"Malformed request to {endpoint} - {req_raw.hex()}") + self.logger.warning(f"Malformed request to {endpoint} - {req_raw.hex()}") return Response() req_header = SaoRequestHeader(req_raw) diff --git a/titles/sao/read.py b/titles/sao/read.py index 6299bc4..0148ed0 100644 --- a/titles/sao/read.py +++ b/titles/sao/read.py @@ -26,7 +26,7 @@ class SaoReader(BaseReader): await self.read_csv(f"{self.bin_dir}") else: - self.logger.warn("Directory not found, nothing to import") + self.logger.warning("Directory not found, nothing to import") def load_csv_file(self, file: str) -> List[Dict]: ret = [] From 399f983bea5a5239e9b794f16ceb826aa1f59445 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sun, 2 Mar 2025 04:10:06 -0500 Subject: [PATCH 081/130] allnet: fix download order response, billing logging --- core/allnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 0b6006b..c878870 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -358,7 +358,7 @@ class AllnetServlet: else: machine = await self.data.arcade.get_machine(req.serial) - if not machine or not machine['ota_enable'] or not machine['is_cab'] or machine['is_blacklisted']: + if not machine or not machine['ota_enable'] or not machine['is_cab']: resp = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n" if is_dfi: return PlainTextResponse( @@ -635,7 +635,7 @@ class BillingServlet: await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, "", log_details, None, machine['arcade'], machine['id'], request_ip, req.gameid, req.gamever) self.logger.info( - f"Unregistered Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount " + f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount " f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}" ) else: From cdd46d51b7434ded4973fafa19ec5f6902cdfdc7 Mon Sep 17 00:00:00 2001 From: Dniel97 Date: Sun, 2 Mar 2025 18:34:06 +0100 Subject: [PATCH 082/130] chuni: fix favorite music list --- titles/chuni/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/titles/chuni/base.py b/titles/chuni/base.py index 5b03c78..9333aab 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -1017,9 +1017,13 @@ class ChuniBase: # New in LUMINOUS PLUS if "userFavoriteMusicList" in upsert: # musicId, orderId - music_ids = set(int(m["musicId"]) for m in upsert["userFavoriteMusicList"]) + music_ids = set( + int(m["musicId"]) + for m in upsert["userFavoriteMusicList"] + if m["musicId"] != "-1" + ) current_favorites = await self.data.item.get_all_favorites( - user_id, self.version, fav_kind=FavoriteItemKind.MUSIC + user_id, self.version, fav_kind=FavoriteItemKind.MUSIC.value ) if current_favorites is None: From 376b77be29c0a51f2aea0e632c8b8abef5eb3a67 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 20 Mar 2025 14:53:28 -0400 Subject: [PATCH 083/130] frontend: serial rollover after 9999 generated serials --- core/frontend.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/frontend.py b/core/frontend.py index 2e2fc2d..8226759 100644 --- a/core/frontend.py +++ b/core/frontend.py @@ -21,6 +21,10 @@ from os import path, environ, mkdir, W_OK, access from core import CoreConfig, Utils from core.data import Data +# A-HJ-NP-Z +SERIAL_LETTERS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'] +ARTEMIS_SERIAL_PREFIX = "A69A" + class PermissionOffset(Enum): USER = 0 # Regular user USERMOD = 1 # Can moderate other users @@ -892,8 +896,13 @@ class FE_System(FE_Base): generated = await self.data.arcade.get_num_generated_keychips() if not generated: generated = 0 - serial = self.data.arcade.format_serial("A69A", 1, "A", generated + 1, int(append)) - serial_dash = self.data.arcade.format_serial("A69A", 1, "A", generated + 1, int(append), True) + + rollover = generated // 9999 + serial_num = (generated % 9999) + 1 + serial_letter = SERIAL_LETTERS[rollover] + + serial_dash = self.data.arcade.format_serial(ARTEMIS_SERIAL_PREFIX, 1, serial_letter, serial_num, int(append), True) + serial = serial_dash.replace("-", "") cab_id = await self.data.arcade.create_machine(int(shopid), serial, None, game_code if game_code else None) From 882560a790a916f09a835e3fb17913a5e77f665c Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Fri, 21 Mar 2025 09:47:44 -0400 Subject: [PATCH 084/130] adb: fix semantics with FelicaLookupEx --- core/adb_handlers/felica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/adb_handlers/felica.py b/core/adb_handlers/felica.py index a037bbe..bd10708 100644 --- a/core/adb_handlers/felica.py +++ b/core/adb_handlers/felica.py @@ -39,7 +39,7 @@ class ADBFelicaLookupExRequest(ADBBaseRequest): def __init__(self, data: bytes) -> None: super().__init__(data) self.random = struct.unpack_from("<16s", data, 0x20)[0] - idm, dfc = struct.unpack_from(">QQ", data, 0x30) + idm, dfc, self.arbitrary = struct.unpack_from(">QH6s", data, 0x30) self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40) self.idm = hex(idm)[2:].upper() self.dfc = hex(dfc)[2:].upper() From afdcd9a7315a18ee59919fbd55cf22efb8643c80 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sat, 22 Mar 2025 00:58:49 -0400 Subject: [PATCH 085/130] mai2: remove print statements from frontend --- titles/mai2/frontend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index f8a7176..c760e13 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -333,7 +333,6 @@ class Mai2Frontend(FE_Base): return RedirectResponse("/game/mai2/", 303) form_data = await request.form() - print(form_data) event_id: int = form_data.get("evtId", None) new_enabled: bool = bool(form_data.get("evtEnabled", False)) try: @@ -341,7 +340,6 @@ class Mai2Frontend(FE_Base): except: new_start_date = None - print(f"{event_id} {new_enabled} {new_start_date}") if event_id is None or new_start_date is None: return RedirectResponse("/game/mai2/events/?e=4", 303) From 20d9a2da9c8f693a7f5e6d17414115d7800c8298 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sat, 22 Mar 2025 00:58:56 -0400 Subject: [PATCH 086/130] sao: fix frontend --- titles/sao/templates/sao_index.jinja | 1 - 1 file changed, 1 deletion(-) diff --git a/titles/sao/templates/sao_index.jinja b/titles/sao/templates/sao_index.jinja index 35d42ce..a42d3ce 100644 --- a/titles/sao/templates/sao_index.jinja +++ b/titles/sao/templates/sao_index.jinja @@ -110,7 +110,6 @@ function toggle_new_name_form() {

    Profile for {{ profile.nick_name }} 

    {% include "core/templates/widgets/err_banner.jinja" %} -{% include "core/templates/widgets/succ_banner.jinja" %}
    +
    + + +
    +
    diff --git a/core/templates/user/index.jinja b/core/templates/user/index.jinja index 88b91c9..79b73b8 100644 --- a/core/templates/user/index.jinja +++ b/core/templates/user/index.jinja @@ -159,19 +159,10 @@ Update successful {% if arcades is defined and arcades|length > 0 %} -

    Arcades

    +

    Arcades you manage

      {% for a in arcades %} -

    • {{ a.name }}

      - {% if a.machines|length > 0 %} - - - {% for m in a.machines %} - - {% endfor %} -
      SerialGameLast Seen
      {{ m.serial }}{{ m.game }}{{ m.last_seen }}
      - {% endif %} -
      • +

    • {{ a.name }}

      • {% endfor %}
    {% endif %} diff --git a/core/templates/widgets/err_banner.jinja b/core/templates/widgets/err_banner.jinja index f1f4899..d36b856 100644 --- a/core/templates/widgets/err_banner.jinja +++ b/core/templates/widgets/err_banner.jinja @@ -27,6 +27,10 @@ Access Denied Card already registered {% elif error == 13 %} AmusementIC Access Codes beginning with 5 must have IDm +{% elif error == 14 %} +Arcade does not exist +{% elif error == 15 %} +Some info failed to update {% else %} An unknown error occoured {% endif %} From 017cdecbaa66bbaa78d68febf0abeae73e7c8d4b Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 25 Mar 2025 11:22:37 -0400 Subject: [PATCH 088/130] db: fix missing param in add_arcade_owner --- core/data/schema/arcade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index 7951ad1..038077d 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -265,7 +265,7 @@ class ArcadeData(BaseData): return result.fetchall() async def add_arcade_owner(self, arcade_id: int, user_id: int, permissions: int = 1) -> Optional[int]: - sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id) + sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id, permissions=permissions) result = await self.execute(sql) if result is None: From c1fa528e45e0821333d39aa1008b1d2b947aa3c8 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 25 Mar 2025 11:30:44 -0400 Subject: [PATCH 089/130] chuni: fix frontend 500 if no profile is available --- core/frontend.py | 2 ++ titles/chuni/frontend.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/core/frontend.py b/core/frontend.py index 63abc7f..47399d2 100644 --- a/core/frontend.py +++ b/core/frontend.py @@ -919,6 +919,8 @@ class FE_System(FE_Base): serial = serial_dash.replace("-", "") cab_id = await self.data.arcade.create_machine(int(shopid), serial, None, game_code if game_code else None) + if cab_id is None: + return RedirectResponse("/sys/?e=4", 303) return Response(template.render( title=f"{self.core_config.server.name} | System", diff --git a/titles/chuni/frontend.py b/titles/chuni/frontend.py index 876e873..d13252c 100644 --- a/titles/chuni/frontend.py +++ b/titles/chuni/frontend.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Tuple, Dict from starlette.routing import Route, Mount from starlette.requests import Request from starlette.responses import Response, RedirectResponse @@ -123,7 +123,7 @@ class ChuniFrontend(FE_Base): if usr_sesh.user_id > 0: versions = await self.data.profile.get_all_profile_versions(usr_sesh.user_id) - profile = [] + profile = None if versions: # chunithm_version is -1 means it is not initialized yet, select a default version from existing. if usr_sesh.chunithm_version < 0: @@ -350,7 +350,9 @@ class ChuniFrontend(FE_Base): else: return RedirectResponse("/gate/", 303) - async def get_available_map_icons(self, version: int, profile: Row) -> (List[dict], int): + async def get_available_map_icons(self, version: int, profile: Row) -> Tuple[List[Dict], int]: + if profile is None: + return ([], 0) items = dict() rows = await self.data.static.get_map_icons(version) if rows is None: @@ -373,7 +375,9 @@ class ChuniFrontend(FE_Base): return (items, len(rows)) - async def get_available_system_voices(self, version: int, profile: Row) -> (List[dict], int): + async def get_available_system_voices(self, version: int, profile: Row) -> Tuple[List[Dict], int]: + if profile is None: + return ([], 0) items = dict() rows = await self.data.static.get_system_voices(version) if rows is None: @@ -396,7 +400,7 @@ class ChuniFrontend(FE_Base): return (items, len(rows)) - async def get_available_nameplates(self, version: int, profile: Row) -> (List[dict], int): + async def get_available_nameplates(self, version: int, profile: Row) -> Tuple[List[Dict], int]: items = dict() rows = await self.data.static.get_nameplates(version) if rows is None: @@ -419,7 +423,7 @@ class ChuniFrontend(FE_Base): return (items, len(rows)) - async def get_available_trophies(self, version: int, profile: Row) -> (List[dict], int): + async def get_available_trophies(self, version: int, profile: Row) -> Tuple[List[Dict], int]: items = dict() rows = await self.data.static.get_trophies(version) if rows is None: @@ -442,7 +446,7 @@ class ChuniFrontend(FE_Base): return (items, len(rows)) - async def get_available_characters(self, version: int, profile: Row) -> (List[dict], int): + async def get_available_characters(self, version: int, profile: Row) -> Tuple[List[Dict], int]: items = dict() rows = await self.data.static.get_characters(version) if rows is None: @@ -465,7 +469,7 @@ class ChuniFrontend(FE_Base): return (items, len(rows)) - async def get_available_avatar_items(self, version: int, category: AvatarCategory, user_unlocked_items: List[int]) -> (List[dict], int): + async def get_available_avatar_items(self, version: int, category: AvatarCategory, user_unlocked_items: List[int]) -> Tuple[List[Dict], int]: items = dict() rows = await self.data.static.get_avatar_items(version, category.value) if rows is None: From 60002a466f46c063ebde0fdc5b275a13415a7fda Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 25 Mar 2025 11:32:12 -0400 Subject: [PATCH 090/130] diva: put full name in frontend header --- titles/diva/templates/diva_header.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titles/diva/templates/diva_header.jinja b/titles/diva/templates/diva_header.jinja index b92379a..acfa4e9 100644 --- a/titles/diva/templates/diva_header.jinja +++ b/titles/diva/templates/diva_header.jinja @@ -1,5 +1,5 @@
    -

    diva

    +

    Project Diva Arcade Future Tone

    • PROFILE
    • RECORD
      • From a2f71dc553fad8472a306f7492bbacb7444dbb72 Mon Sep 17 00:00:00 2001 From: Dniel97 Date: Wed, 26 Mar 2025 15:22:55 +0100 Subject: [PATCH 091/130] ongeki: bright MEMORY Act.3 support added --- changelog.md | 4 ++++ docs/game_specific_info.md | 5 ++++- example_config/cardmaker.yaml | 6 +++--- example_config/ongeki.yaml | 22 ++++++++++++++++------ readme.md | 1 + titles/ongeki/bright.py | 2 +- titles/ongeki/brightmemoryact3.py | 24 ++++++++++++++++++++++++ titles/ongeki/const.py | 18 ++++++++++-------- titles/ongeki/index.py | 26 +++++++++++++++----------- titles/ongeki/read.py | 3 ++- titles/ongeki/schema/item.py | 14 +++++++++++--- 11 files changed, 91 insertions(+), 34 deletions(-) create mode 100644 titles/ongeki/brightmemoryact3.py diff --git a/changelog.md b/changelog.md index a1e3f85..4620084 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,10 @@ # Changelog Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to. +## 20250327 ++ O.N.G.E.K.I. bright MEMORY Act.3 support added ++ CardMaker support updated + ## 20240811 ### System + Change backend from Twisted to Starlette diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 16c15e7..7337191 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -463,7 +463,10 @@ After that, on next login the present should be received (or whenever it suppose * FESTiVAL: Yes (added in A031) * FESTiVAL PLUS: Yes (added in A035) * BUDDiES: Yes (added in A039) -* O.N.G.E.K.I. bright MEMORY: Yes + * BUDDiES PLUS: Yes (added in A047) +* O.N.G.E.K.I.: + * bright MEMORY: Yes + * bright MEMORY Act.3 (added in A046) ### Importer diff --git a/example_config/cardmaker.yaml b/example_config/cardmaker.yaml index fb17756..b88f75d 100644 --- a/example_config/cardmaker.yaml +++ b/example_config/cardmaker.yaml @@ -8,6 +8,6 @@ version: chuni: 2.00.00 maimai: 1.20.00 1: - ongeki: 1.35.03 - chuni: 2.10.00 - maimai: 1.30.00 \ No newline at end of file + ongeki: 1.45.01 + chuni: 2.25.00 + maimai: 1.45.00 diff --git a/example_config/ongeki.yaml b/example_config/ongeki.yaml index 9af5efe..9ed04a6 100644 --- a/example_config/ongeki.yaml +++ b/example_config/ongeki.yaml @@ -7,12 +7,12 @@ gachas: enabled_gachas: - 1011 - 1012 - - 1043 - - 1067 - - 1068 - - 1069 - - 1070 - - 1071 + # - 1043 + # - 1067 + # - 1068 + # - 1069 + # - 1070 + # - 1071 - 1072 - 1073 - 1074 @@ -30,12 +30,22 @@ gachas: - 1156 - 1163 - 1164 + # 5th anniversary gacha + - 1165 + # 2024 gacha + - 1166 + # 6th anniversary gacha + - 1167 + # 2025 gacha + - 1168 version: 6: card_maker: 1.30.01 7: card_maker: 1.35.03 + 8: + card_maker: 1.45.01 crypto: encrypted_only: False \ No newline at end of file diff --git a/readme.md b/readme.md index e44c137..57f81a4 100644 --- a/readme.md +++ b/readme.md @@ -60,6 +60,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + R.E.D. PLUS + bright + bright MEMORY + + bright MEMORY Act.3 + POKKÉN TOURNAMENT + Final Online diff --git a/titles/ongeki/bright.py b/titles/ongeki/bright.py index 5c95af3..c8979cb 100644 --- a/titles/ongeki/bright.py +++ b/titles/ongeki/bright.py @@ -196,7 +196,7 @@ class OngekiBright(OngekiBase): # make sure to only show gachas for the current version # so only up to bright, 1140 is the first bright memory gacha - if self.version == OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY: + if self.version >= OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY: game_gacha_list.append(tmp) elif ( self.version == OngekiConstants.VER_ONGEKI_BRIGHT diff --git a/titles/ongeki/brightmemoryact3.py b/titles/ongeki/brightmemoryact3.py new file mode 100644 index 0000000..4c0cf16 --- /dev/null +++ b/titles/ongeki/brightmemoryact3.py @@ -0,0 +1,24 @@ +from typing import Dict + +from core.config import CoreConfig +from titles.ongeki.brightmemory import OngekiBrightMemory +from titles.ongeki.const import OngekiConstants +from titles.ongeki.config import OngekiConfig + + +class OngekiBrightMemoryAct3(OngekiBrightMemory): + def __init__(self, core_cfg: CoreConfig, game_cfg: OngekiConfig) -> None: + super().__init__(core_cfg, game_cfg) + self.version = OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY_ACT3 + + async def handle_get_game_setting_api_request(self, data: Dict) -> Dict: + ret = await super().handle_get_game_setting_api_request(data) + ret["gameSetting"]["dataVersion"] = "1.45.00" + ret["gameSetting"]["onlineDataVersion"] = "1.45.00" + ret["gameSetting"]["maxCountCharacter"] = 50 + ret["gameSetting"]["maxCountCard"] = 300 + ret["gameSetting"]["maxCountItem"] = 300 + ret["gameSetting"]["maxCountMusic"] = 50 + ret["gameSetting"]["maxCountMusicItem"] = 300 + ret["gameSetting"]["maxCountRivalMusic"] = 300 + return ret diff --git a/titles/ongeki/const.py b/titles/ongeki/const.py index a7658f7..71ea7f2 100644 --- a/titles/ongeki/const.py +++ b/titles/ongeki/const.py @@ -15,6 +15,7 @@ class OngekiConstants: VER_ONGEKI_RED_PLUS = 5 VER_ONGEKI_BRIGHT = 6 VER_ONGEKI_BRIGHT_MEMORY = 7 + VER_ONGEKI_BRIGHT_MEMORY_ACT3 = 8 EVT_TYPES: Enum = Enum( "EVT_TYPES", @@ -94,14 +95,15 @@ class OngekiConstants: Lunatic = 10 VERSION_NAMES = ( - "ONGEKI", - "ONGEKI +", - "ONGEKI SUMMER", - "ONGEKI SUMMER +", - "ONGEKI R.E.D.", - "ONGEKI R.E.D. +", - "ONGEKI bright", - "ONGEKI bright MEMORY", + "O.N.G.E.K.I.", + "O.N.G.E.K.I. PLUS", + "O.N.G.E.K.I. SUMMER", + "O.N.G.E.K.I. SUMMER PLUS", + "O.N.G.E.K.I. R.E.D.", + "O.N.G.E.K.I. R.E.D. PLUS", + "O.N.G.E.K.I. bright", + "O.N.G.E.K.I. bright MEMORY", + "O.N.G.E.K.I. bright MEMORY Act.3", ) @classmethod diff --git a/titles/ongeki/index.py b/titles/ongeki/index.py index 3bd0e15..3960cfa 100644 --- a/titles/ongeki/index.py +++ b/titles/ongeki/index.py @@ -29,6 +29,7 @@ from .red import OngekiRed from .redplus import OngekiRedPlus from .bright import OngekiBright from .brightmemory import OngekiBrightMemory +from .brightmemoryact3 import OngekiBrightMemoryAct3 class OngekiServlet(BaseServlet): @@ -50,6 +51,7 @@ class OngekiServlet(BaseServlet): OngekiRedPlus(core_cfg, self.game_cfg), OngekiBright(core_cfg, self.game_cfg), OngekiBrightMemory(core_cfg, self.game_cfg), + OngekiBrightMemoryAct3(core_cfg, self.game_cfg), ] self.logger = logging.getLogger("ongeki") @@ -150,26 +152,28 @@ class OngekiServlet(BaseServlet): return Response(zlib.compress(b'{"returnCode": 1}')) req_raw = await request.body() - encrtped = False + encrypted = False internal_ver = 0 client_ip = Utils.get_ip_addr(request) if version < 105: # 1.0 internal_ver = OngekiConstants.VER_ONGEKI - elif version >= 105 and version < 110: # Plus + elif version >= 105 and version < 110: # PLUS internal_ver = OngekiConstants.VER_ONGEKI_PLUS - elif version >= 110 and version < 115: # Summer + elif version >= 110 and version < 115: # SUMMER internal_ver = OngekiConstants.VER_ONGEKI_SUMMER - elif version >= 115 and version < 120: # Summer Plus + elif version >= 115 and version < 120: # SUMMER PLUS internal_ver = OngekiConstants.VER_ONGEKI_SUMMER_PLUS - elif version >= 120 and version < 125: # Red + elif version >= 120 and version < 125: # R.E.D. internal_ver = OngekiConstants.VER_ONGEKI_RED - elif version >= 125 and version < 130: # Red Plus + elif version >= 125 and version < 130: # R.E.D. PLUS internal_ver = OngekiConstants.VER_ONGEKI_RED_PLUS - elif version >= 130 and version < 135: # Bright + elif version >= 130 and version < 135: # bright internal_ver = OngekiConstants.VER_ONGEKI_BRIGHT - elif version >= 135 and version < 145: # Bright Memory + elif version >= 135 and version < 145: # bright MEMORY internal_ver = OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY + elif version >= 145: # bright MEMORY Act 3 + internal_ver = OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY_ACT3 if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32: # If we get a 32 character long hex string, it's a hash and we're @@ -204,10 +208,10 @@ class OngekiServlet(BaseServlet): ) return Response(zlib.compress(b'{"stat": "0"}')) - encrtped = True + encrypted = True if ( - not encrtped + not encrypted and self.game_cfg.crypto.encrypted_only and version >= 120 ): @@ -258,7 +262,7 @@ class OngekiServlet(BaseServlet): resp_raw = json.dumps(resp, ensure_ascii=False).encode("utf-8") zipped = zlib.compress(resp_raw) - if not encrtped or version < 120: + if not encrypted or version < 120: if version < 105: return Response(resp_raw) return Response(zipped) diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index 18f1e8a..dff5cf8 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -37,7 +37,8 @@ class OngekiReader(BaseReader): "1025": OngekiConstants.VER_ONGEKI_RED_PLUS, "1030": OngekiConstants.VER_ONGEKI_BRIGHT, "1035": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, - "1040": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY + "1040": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, + "1045": OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY_ACT3, } node = troot.find("VersionID").find("id") diff --git a/titles/ongeki/schema/item.py b/titles/ongeki/schema/item.py index 274e16d..6145116 100644 --- a/titles/ongeki/schema/item.py +++ b/titles/ongeki/schema/item.py @@ -578,7 +578,11 @@ class OngekiItemData(BaseData): return result.lastrowid async def get_mission_points(self, version: int, aime_id: int) -> Optional[List[Dict]]: - sql = select(mission_point).where(and_(mission_point.c.user == aime_id, mission_point.c.version == version)) + sql = select(mission_point).where(and_( + mission_point.c.user == aime_id, + mission_point.c.version <= version) + ).order_by(mission_point.c.version.desc()) + result = await self.execute(sql) if result is None: @@ -702,7 +706,11 @@ class OngekiItemData(BaseData): return result.lastrowid async def get_tech_event(self, version: int, aime_id: int) -> Optional[List[Dict]]: - sql = select(tech_event).where(and_(tech_event.c.user == aime_id, tech_event.c.version == version)) + sql = select(tech_event).where(and_( + tech_event.c.user == aime_id, + tech_event.c.version <= version) + ).order_by(tech_event.c.version.desc()) + result = await self.execute(sql) if result is None: @@ -794,7 +802,7 @@ class OngekiItemData(BaseData): async def get_ranking_event_ranks(self, version: int, aime_id: int) -> Optional[List[Dict]]: # Calculates player rank on GameRequest from server, and sends it back, official spec would rank players in maintenance period, on TODO list - sql = select(event_point.c.id, event_point.c.user, event_point.c.eventId, event_point.c.type, func.row_number().over(partition_by=event_point.c.eventId, order_by=event_point.c.point.desc()).label('rank'), event_point.c.date, event_point.c.point).where(event_point.c.version == version) + sql = select(event_point.c.id, event_point.c.user, event_point.c.eventId, event_point.c.type, func.row_number().over(partition_by=event_point.c.eventId, order_by=event_point.c.point.desc()).label('rank'), event_point.c.date, event_point.c.point).where(event_point.c.version <= version).order_by(event_point.c.version.desc()) result = await self.execute(sql) if result is None: self.logger.error(f"failed to rank aime_id: {aime_id} ranking event positions") From fbcc53aeae35ef3bc2b9e588e8f07bfe84e7ac53 Mon Sep 17 00:00:00 2001 From: Dniel97 Date: Wed, 26 Mar 2025 20:50:30 +0100 Subject: [PATCH 092/130] ongeki: update ongeki_static_tech_music_uk --- ...geki_update_ongeki_static_tech_music_uk.py | 30 +++++++++++++++++++ titles/ongeki/read.py | 3 ++ titles/ongeki/schema/static.py | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py diff --git a/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py b/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py new file mode 100644 index 0000000..c37f8a8 --- /dev/null +++ b/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py @@ -0,0 +1,30 @@ +"""ONGEKI update ongeki_static_tech_music_uk + +Revision ID: 1d0014d35220 +Revises: 9c42e54a27fe +Create Date: 2025-03-26 20:44:55.590992 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '1d0014d35220' +down_revision = '9c42e54a27fe' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('ongeki_static_tech_music_uk', 'ongeki_static_tech_music', type_='unique') + op.create_unique_constraint('ongeki_static_tech_music_uk', 'ongeki_static_tech_music', ['version', 'eventId', 'musicId', 'level']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('ongeki_static_tech_music_uk', 'ongeki_static_tech_music', type_='unique') + op.create_unique_constraint('ongeki_static_tech_music_uk', 'ongeki_static_tech_music', ['version', 'musicId']) + # ### end Alembic commands ### diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index dff5cf8..ed3043f 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -137,6 +137,9 @@ class OngekiReader(BaseReader): troot.find("EventType").text ].value + if troot.find("EventType").text == "MissionEvent": + name = (troot.find("Event").find("MissionName").find("str").text) + await self.data.static.put_event(self.version, id, event_type, name) self.logger.info(f"Added event {id}") diff --git a/titles/ongeki/schema/static.py b/titles/ongeki/schema/static.py index 30b2767..5d6a3e0 100644 --- a/titles/ongeki/schema/static.py +++ b/titles/ongeki/schema/static.py @@ -135,7 +135,7 @@ tech_music = Table( Column("eventId", Integer, nullable=False), Column("musicId", Integer, nullable=False), Column("level", Integer, nullable=False), - UniqueConstraint("version", "musicId", name="ongeki_static_tech_music_uk"), + UniqueConstraint("version", "eventId", "musicId", "level", name="ongeki_static_tech_music_uk"), mysql_charset="utf8mb4", ) From f939d4976ebd4404667dcb3cd455bf137961856f Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sat, 29 Mar 2025 11:22:12 -0400 Subject: [PATCH 093/130] chuni: make total columns BigInt, for #203 --- .../91c682918b67_chuni_fix_total_scores.py | 92 +++++++++++++++++++ titles/chuni/schema/profile.py | 17 ++-- 2 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 core/data/alembic/versions/91c682918b67_chuni_fix_total_scores.py diff --git a/core/data/alembic/versions/91c682918b67_chuni_fix_total_scores.py b/core/data/alembic/versions/91c682918b67_chuni_fix_total_scores.py new file mode 100644 index 0000000..685f19a --- /dev/null +++ b/core/data/alembic/versions/91c682918b67_chuni_fix_total_scores.py @@ -0,0 +1,92 @@ +"""chuni_fix_total_scores + +Revision ID: 91c682918b67 +Revises: 9c42e54a27fe +Create Date: 2025-03-29 11:19:46.063173 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '91c682918b67' +down_revision = '9c42e54a27fe' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('chuni_profile_data', 'totalMapNum', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalHiScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalBasicHighScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalExpertHighScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalMasterHighScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalRepertoireCount', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalAdvancedHighScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalUltimaHighScore', + existing_type=mysql.INTEGER(display_width=11), + type_=sa.BigInteger(), + existing_nullable=True, + existing_server_default=sa.text("'0'")) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('chuni_profile_data', 'totalUltimaHighScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True, + existing_server_default=sa.text("'0'")) + op.alter_column('chuni_profile_data', 'totalAdvancedHighScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalRepertoireCount', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalMasterHighScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalExpertHighScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalBasicHighScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalHiScore', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + op.alter_column('chuni_profile_data', 'totalMapNum', + existing_type=sa.BigInteger(), + type_=mysql.INTEGER(display_width=11), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/titles/chuni/schema/profile.py b/titles/chuni/schema/profile.py index 1362c9f..c7fb750 100644 --- a/titles/chuni/schema/profile.py +++ b/titles/chuni/schema/profile.py @@ -35,13 +35,13 @@ profile = Table( Column("friendCount", Integer), Column("lastPlaceId", Integer), Column("nameplateId", Integer), - Column("totalMapNum", Integer), + Column("totalMapNum", BigInteger), Column("lastAllNetId", Integer), Column("lastClientId", String(25)), Column("lastPlayDate", String(25)), Column("lastRegionId", Integer), Column("playerRating", Integer), - Column("totalHiScore", Integer), + Column("totalHiScore", BigInteger), Column("webLimitDate", String(25)), Column("firstPlayDate", String(25)), Column("highestRating", Integer), @@ -59,12 +59,12 @@ profile = Table( Column("firstDataVersion", String(25)), Column("reincarnationNum", Integer), Column("playedTutorialBit", Integer), - Column("totalBasicHighScore", Integer), - Column("totalExpertHighScore", Integer), - Column("totalMasterHighScore", Integer), - Column("totalRepertoireCount", Integer), + Column("totalBasicHighScore", BigInteger), + Column("totalExpertHighScore", BigInteger), + Column("totalMasterHighScore", BigInteger), + Column("totalRepertoireCount", BigInteger), Column("firstTutorialCancelNum", Integer), - Column("totalAdvancedHighScore", Integer), + Column("totalAdvancedHighScore", BigInteger), Column("masterTutorialCancelNum", Integer), Column("ext1", Integer), # Added in chunew Column("ext2", Integer), @@ -111,7 +111,7 @@ profile = Table( Column("classEmblemBase", Integer, server_default="0"), Column("battleRankPoint", Integer, server_default="0"), Column("netBattle2ndCount", Integer, server_default="0"), - Column("totalUltimaHighScore", Integer, server_default="0"), + Column("totalUltimaHighScore", BigInteger, server_default="0"), Column("skillId", Integer, server_default="0"), Column("lastCountryCode", String(5), server_default="JPN"), Column("isNetBattleHost", Boolean, server_default="0"), @@ -808,6 +808,7 @@ class ChuniProfileData(BaseData): if result is None: return None return result.fetchone() + async def get_overview(self) -> Dict: # Fetch and add up all the playcounts playcount_sql = await self.execute(select(profile.c.playCount)) From 96a252cbf357c5a9dcb1aaced2eeaccf644acf00 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sat, 29 Mar 2025 11:25:07 -0400 Subject: [PATCH 094/130] ongeki: fix act 3 database upgrade script --- .../1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py b/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py index c37f8a8..c209035 100644 --- a/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py +++ b/core/data/alembic/versions/1d0014d35220_ongeki_update_ongeki_static_tech_music_uk.py @@ -1,7 +1,7 @@ """ONGEKI update ongeki_static_tech_music_uk Revision ID: 1d0014d35220 -Revises: 9c42e54a27fe +Revises: 91c682918b67 Create Date: 2025-03-26 20:44:55.590992 """ @@ -11,7 +11,7 @@ from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. revision = '1d0014d35220' -down_revision = '9c42e54a27fe' +down_revision = '91c682918b67' branch_labels = None depends_on = None From 1d545b2bd26601d54b831289d4c03deaa4a3c51e Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 02:32:34 +0800 Subject: [PATCH 095/130] add const.py of prism --- titles/mai2/const.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 0d13a0d..1d6a4dd 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -56,6 +56,7 @@ class Mai2Constants: VER_MAIMAI_DX_FESTIVAL_PLUS = 20 VER_MAIMAI_DX_BUDDIES = 21 VER_MAIMAI_DX_BUDDIES_PLUS = 22 + VER_MAIMAI_DX_PRISM = 23 VERSION_STRING = ( "maimai", @@ -80,7 +81,8 @@ class Mai2Constants: "maimai DX FESTiVAL", "maimai DX FESTiVAL PLUS", "maimai DX BUDDiES", - "maimai DX BUDDiES PLUS" + "maimai DX BUDDiES PLUS", + "maimai DX PRiSM" ) @classmethod From fa94c029ca936ae12f4a1e5b5c3e9ccaf4f3803e Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 03:07:35 +0800 Subject: [PATCH 096/130] add GetUserNewItemListApi handler --- titles/mai2/index.py | 17 ++++++++++++++--- titles/mai2/prism.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 titles/mai2/prism.py diff --git a/titles/mai2/index.py b/titles/mai2/index.py index e8b88ec..86923e7 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -31,6 +31,7 @@ from .festival import Mai2Festival from .festivalplus import Mai2FestivalPlus from .buddies import Mai2Buddies from .buddiesplus import Mai2BuddiesPlus +from .prism import Mai2Prism class Mai2Servlet(BaseServlet): @@ -66,7 +67,8 @@ class Mai2Servlet(BaseServlet): Mai2Festival, Mai2FestivalPlus, Mai2Buddies, - Mai2BuddiesPlus + Mai2BuddiesPlus, + Mai2Prism ] self.logger = logging.getLogger("mai2") @@ -306,8 +308,11 @@ class Mai2Servlet(BaseServlet): internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES - elif version >= 145: # BUDDiES PLUS - internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS + elif version >= 145 and version <150: # BUDDiES PLUS + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS, + elif version >=150: + internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM + elif game_code == "SDGA": # Int if version < 105: # 1.0 internal_ver = Mai2Constants.VER_MAIMAI_DX @@ -325,6 +330,12 @@ class Mai2Servlet(BaseServlet): internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL elif version >= 135 and version < 140: # FESTiVAL PLUS internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS + elif version >= 140 and version < 145: # BUDDiES + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES + elif version >= 145 and version <150: # BUDDiES PLUS + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS, + elif version >=150: + internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32: # If we get a 32 character long hex string, it's a hash and we're diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py new file mode 100644 index 0000000..c208a85 --- /dev/null +++ b/titles/mai2/prism.py @@ -0,0 +1,25 @@ +from typing import Dict + +from core.config import CoreConfig +from titles.mai2.buddiesplus import Mai2BuddiesPlus +from titles.mai2.const import Mai2Constants +from titles.mai2.config import Mai2Config + +class Mai2Prism(Mai2BuddiesPlus): + def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None: + super().__init__(cfg, game_cfg) + self.version = Mai2Constants.VER_MAIMAI_DX_PRISM + + async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict: + user_data = await super().handle_cm_get_user_preview_api_request(data) + + # hardcode lastDataVersion for CardMaker + user_data["lastDataVersion"] = "1.50.00" + return user_data + + + async def handle_get_user_new_item_list_api_request(self, data: Dict) -> Dict: + return { + "user_id": data["userId"], + "userItemList": [] + } \ No newline at end of file From f4b9f48ed62e54f4ca038d4945f873783b33323e Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 03:07:59 +0800 Subject: [PATCH 097/130] add cardmaker support for Prism --- titles/cm/read.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/cm/read.py b/titles/cm/read.py index b4b3b5e..cf697c6 100644 --- a/titles/cm/read.py +++ b/titles/cm/read.py @@ -207,7 +207,8 @@ class CardMakerReader(BaseReader): "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL, "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS, "1.40": Mai2Constants.VER_MAIMAI_DX_BUDDIES, - "1.45": Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS + "1.45": Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS, + "1.50": Mai2Constants.VER_MAIMAI_DX_PRISM } for root, dirs, files in os.walk(base_dir): From 6b1b607db068c39b1001c1d295128eb1302bc1ed Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 04:35:14 +0800 Subject: [PATCH 098/130] add GetGameMusicScoreApi handler --- titles/mai2/prism.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index c208a85..30044df 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -22,4 +22,15 @@ class Mai2Prism(Mai2BuddiesPlus): return { "user_id": data["userId"], "userItemList": [] - } \ No newline at end of file + } + + #seems to be used for downloading music scores online + async def handle_get_game_music_score_api_request(self, data: Dict) -> Dict: + return { + "gameMusicScore": { + "musicId": data["musicId"], + "level": data["level"], + "type": data["type"], + "scoreData": "" + } + } From 6821ab6f4668b89974998fbd494eb995097a6e40 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 04:35:39 +0800 Subject: [PATCH 099/130] add UploadUserPlaylogListApi handler for Exp version --- titles/mai2/dx.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index b37a3f4..01e440f 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -112,6 +112,17 @@ class Mai2DX(Mai2Base): return {"returnCode": 1, "apiName": "UploadUserPlaylogApi"} + # Exp version use this instead of UploadUserPlaylogApi in 1.50 + async def handle_upload_user_playlog_list_api_request(self, data: Dict) -> Dict: + user_id = data["userId"] + playlog_list = data["userPlaylogList"] + + for playlog in playlog_list: + await self.data.score.put_playlog(user_id, playlog) + + return {"returnCode": 1, "apiName": "UploadUserPlaylogApi"} + + async def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict: user_id = data["userId"] charge = data["userCharge"] From 814c4fd28407ff6c2e6c82f0bfce2d9a21777a83 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 04:54:01 +0800 Subject: [PATCH 100/130] add GetGameKaleidxScopeApi handler --- titles/mai2/prism.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index 30044df..addb646 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -34,3 +34,15 @@ class Mai2Prism(Mai2BuddiesPlus): "scoreData": "" } } + + async def handle_get_game_kaleidx_scope_api_request(self, data: Dict) -> Dict: + return { + "gameKaleidxScopeList": [ + {"gateId": 1, "phaseId": 6}, + {"gateId": 2, "phaseId": 6}, + {"gateId": 3, "phaseId": 6}, + {"gateId": 4, "phaseId": 6}, + {"gateId": 5, "phaseId": 6}, + {"gateId": 6, "phaseId": 6} + ] + } \ No newline at end of file From 36354ae109d5a4c11ef66cb496ab972f33edff68 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 05:46:44 +0800 Subject: [PATCH 101/130] add GetUserKaleidxScopeApi handler --- titles/mai2/prism.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index addb646..c826330 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -45,4 +45,13 @@ class Mai2Prism(Mai2BuddiesPlus): {"gateId": 5, "phaseId": 6}, {"gateId": 6, "phaseId": 6} ] + } + + async def handle_get_user_kaleidx_scope_api_request(self, data: Dict) -> Dict: + user_id = data["userId"] + + + return { + "userId": user_id, + "userKaleidxScopeList": [] } \ No newline at end of file From d77d02c2dde132025773abae468e2cb4eab5091d Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 06:42:28 +0800 Subject: [PATCH 102/130] database add Mai2Prism support --- .../d0f1c7fa9505_mai2_add_prism_support.py | 28 +++++++++++++++++++ titles/mai2/schema/score.py | 1 + 2 files changed, 29 insertions(+) create mode 100644 core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py diff --git a/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py b/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py new file mode 100644 index 0000000..c879706 --- /dev/null +++ b/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py @@ -0,0 +1,28 @@ +"""Mai2 add PRiSM support + +Revision ID: d0f1c7fa9505 +Revises: 1d0014d35220 +Create Date: 2025-04-02 06:37:10.657372 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd0f1c7fa9505' +down_revision = '1d0014d35220' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('mai2_playlog', sa.Column('extBool2', sa.Boolean(), nullable=True,server_default=sa.text("NULL"))) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('mai2_playlog', 'extBool2') + # ### end Alembic commands ### diff --git a/titles/mai2/schema/score.py b/titles/mai2/schema/score.py index cbe7448..56957b3 100644 --- a/titles/mai2/schema/score.py +++ b/titles/mai2/schema/score.py @@ -147,6 +147,7 @@ playlog = Table( Column("extNum2", Integer), Column("extNum4", Integer), Column("extBool1", Boolean), # new with buddies + Column("extBool2", Boolean), # new with prism Column("trialPlayAchievement", Integer), mysql_charset="utf8mb4", ) From 3d84e328928e8fef9e7964a1ad58e4e3b2e497d5 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 09:42:08 +0800 Subject: [PATCH 103/130] add Kaleidx Scope Support --- titles/mai2/dx.py | 6 +++++ titles/mai2/prism.py | 16 ++++++++++--- titles/mai2/schema/score.py | 48 +++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 01e440f..82e99f9 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -282,6 +282,12 @@ class Mai2DX(Mai2Base): for intimate in upsert["userIntimateList"]: await self.data.profile.put_intimacy(user_id, intimate["partnerId"], intimate["intimateLevel"], intimate["intimateCountRewarded"]) + # added in PRiSM + if "userKaleidxScopeList" in upsert and len(upsert["userKaleidxScopeList"]) > 0: + for kaleidx_scope in upsert["userKaleidxScopeList"]: + await self.data.score.put_user_kaleidx_scope(user_id, kaleidx_scope) + + return {"returnCode": 1, "apiName": "UpsertUserAllApi"} async def handle_get_user_data_api_request(self, data: Dict) -> Dict: diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index c826330..5d0da41 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -4,6 +4,8 @@ from core.config import CoreConfig from titles.mai2.buddiesplus import Mai2BuddiesPlus from titles.mai2.const import Mai2Constants from titles.mai2.config import Mai2Config +from titles.mai2.schema.score import kaleidx_scope + class Mai2Prism(Mai2BuddiesPlus): def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None: @@ -48,10 +50,18 @@ class Mai2Prism(Mai2BuddiesPlus): } async def handle_get_user_kaleidx_scope_api_request(self, data: Dict) -> Dict: - user_id = data["userId"] + kaleidx_scope = await self.data.score.get_user_kaleidx_scope_list(data["userId"]) + if kaleidx_scope is None: + return {"userId": data["userId"], "userKaleidxScopeList":[]} + kaleidx_scope_list = [] + for kaleidx_scope_data in kaleidx_scope: + tmp = kaleidx_scope_data._asdict() + tmp.pop("user") + tmp.pop("id") + kaleidx_scope_list.append(tmp) return { - "userId": user_id, - "userKaleidxScopeList": [] + "userId": data["userId"], + "userKaleidxScopeList": kaleidx_scope_list } \ No newline at end of file diff --git a/titles/mai2/schema/score.py b/titles/mai2/schema/score.py index 56957b3..f3e7002 100644 --- a/titles/mai2/schema/score.py +++ b/titles/mai2/schema/score.py @@ -1,3 +1,4 @@ +from configparser import Interpolation from typing import Dict, List, Optional from sqlalchemy import Column, Table, UniqueConstraint, and_ @@ -174,6 +175,34 @@ playlog_2p = Table( mysql_charset="utf8mb4", ) +kaleidx_scope = Table( + "mai2_score_kaleidx_scope", + metadata, + Column("id", Integer, primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("gateId", Integer), + Column("isGateFound", Boolean), + Column("isKeyFound", Boolean), + Column("isClear", Boolean), + Column("totalRestLife", Integer), + Column("totalAchievement", Integer), + Column("totalDeluxscore", Integer), + Column("bestAchievement", Integer), + Column("bestDeluxscore", Integer), + Column("bestAchievementDate", String(25)), + Column("bestDeluxscoreDate", String(25)), + Column("playCount", Integer), + Column("clearDate", String(25)), + Column("lastPlayDate", String(25)), + Column("isInfoWatched", Boolean), + UniqueConstraint("user", "gateId", name="mai2_score_best_uk"), + mysql_charset="utf8mb4" +) + course = Table( "mai2_score_course", metadata, @@ -451,3 +480,22 @@ class Mai2ScoreData(BaseData): self.logger.warning(f"aime_id {aime_id} has no playlog ") return None return result.scalar() + + async def get_user_kaleidx_scope_list(self, user_id: int) -> Optional[List[Row]]: + sql = kaleidx_scope.select(kaleidx_scope.c.user == user_id) + result = await self.execute(sql) + if result is None: + return None + return result.fetchall() + + async def put_user_kaleidx_scope(self, user_id: int, user_kaleidx_scope_data: Dict) -> Optional[int]: + user_kaleidx_scope_data["user"] = user_id + sql = insert(kaleidx_scope).values(**user_kaleidx_scope_data) + + conflict = sql.on_duplicate_key_update(**user_kaleidx_scope_data) + + result = await self.execute(conflict) + if result is None: + self.logger.error(f"put_user_kaleidx_scope: Failed to insert! user_id {user_id}") + return None + return result.lastrowid \ No newline at end of file From 94b3c47c3c35dfcc5bbeffdbcc2f5898aa24d4d5 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 09:42:40 +0800 Subject: [PATCH 104/130] update readme.md and game_specific_info.md --- docs/game_specific_info.md | 4 +++- readme.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 7337191..6bb3f67 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -203,7 +203,7 @@ Presents are items given to the user when they login, with a little animation (f ### Versions | Game Code | Version ID | Version Name | -| --------- | ---------- | ----------------------- | +|-----------|------------|-------------------------| | SBXL | 0 | maimai | | SBXL | 1 | maimai PLUS | | SBZF | 2 | maimai GreeN | @@ -227,6 +227,8 @@ Presents are items given to the user when they login, with a little animation (f | SDEZ | 20 | maimai DX FESTiVAL PLUS | | SDEZ | 21 | maimai DX BUDDiES | | SDEZ | 22 | maimai DX BUDDiES PLUS | +| SDEZ | 23 | maimai DX PRiSM | + ### Importer diff --git a/readme.md b/readme.md index 57f81a4..e29784d 100644 --- a/readme.md +++ b/readme.md @@ -52,6 +52,7 @@ Games listed below have been tested and confirmed working. Only game versions ol + FESTiVAL PLUS + BUDDiES + BUDDiES PLUS + + PRiSM + O.N.G.E.K.I. + SUMMER From cc7afa6b67dbd8f763ce31568625433ec494f97e Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Wed, 2 Apr 2025 11:57:08 +0800 Subject: [PATCH 105/130] database add kaleidx scope support --- ...16f34bf7b968_mai2_kaleidx_scope_support.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py diff --git a/core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py b/core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py new file mode 100644 index 0000000..b8baa1a --- /dev/null +++ b/core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py @@ -0,0 +1,50 @@ +"""Mai2 Kaleidx Scope Support + +Revision ID: 16f34bf7b968 +Revises: d0f1c7fa9505 +Create Date: 2025-04-02 07:06:15.829591 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '16f34bf7b968' +down_revision = 'd0f1c7fa9505' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mai2_score_kaleidx_scope', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user', sa.Integer(), nullable=False), + sa.Column('gateId', sa.Integer(), nullable=True), + sa.Column('isGateFound', sa.Boolean(), nullable=True), + sa.Column('isKeyFound', sa.Boolean(), nullable=True), + sa.Column('isClear', sa.Boolean(), nullable=True), + sa.Column('totalRestLife', sa.Integer(), nullable=True), + sa.Column('totalAchievement', sa.Integer(), nullable=True), + sa.Column('totalDeluxscore', sa.Integer(), nullable=True), + sa.Column('bestAchievement', sa.Integer(), nullable=True), + sa.Column('bestDeluxscore', sa.Integer(), nullable=True), + sa.Column('bestAchievementDate', sa.String(length=25), nullable=True), + sa.Column('bestDeluxscoreDate', sa.String(length=25), nullable=True), + sa.Column('playCount', sa.Integer(), nullable=True), + sa.Column('clearDate', sa.String(length=25), nullable=True), + sa.Column('lastPlayDate', sa.String(length=25), nullable=True), + sa.Column('isInfoWatched', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['user'], ['aime_user.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user', 'gateId', name='mai2_score_best_uk'), + mysql_charset='utf8mb4' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('mai2_score_kaleidx_scope') + # ### end Alembic commands ### From 9a7fc007bc9cf8046bed792ef110d82e67874177 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Fri, 4 Apr 2025 05:42:53 +0800 Subject: [PATCH 106/130] standardization KaleidxScope variable names --- titles/mai2/dx.py | 4 ++-- titles/mai2/prism.py | 15 +++++++-------- titles/mai2/schema/score.py | 18 +++++++++--------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 82e99f9..9b8b547 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -284,8 +284,8 @@ class Mai2DX(Mai2Base): # added in PRiSM if "userKaleidxScopeList" in upsert and len(upsert["userKaleidxScopeList"]) > 0: - for kaleidx_scope in upsert["userKaleidxScopeList"]: - await self.data.score.put_user_kaleidx_scope(user_id, kaleidx_scope) + for kaleidxscope in upsert["userKaleidxScopeList"]: + await self.data.score.put_user_kaleidxscope(user_id, kaleidxscope) return {"returnCode": 1, "apiName": "UpsertUserAllApi"} diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index 5d0da41..5db7c8a 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -4,7 +4,6 @@ from core.config import CoreConfig from titles.mai2.buddiesplus import Mai2BuddiesPlus from titles.mai2.const import Mai2Constants from titles.mai2.config import Mai2Config -from titles.mai2.schema.score import kaleidx_scope class Mai2Prism(Mai2BuddiesPlus): @@ -50,18 +49,18 @@ class Mai2Prism(Mai2BuddiesPlus): } async def handle_get_user_kaleidx_scope_api_request(self, data: Dict) -> Dict: - kaleidx_scope = await self.data.score.get_user_kaleidx_scope_list(data["userId"]) + kaleidxscope = await self.data.score.get_user_kaleidxscope_list(data["userId"]) - if kaleidx_scope is None: + if kaleidxscope is None: return {"userId": data["userId"], "userKaleidxScopeList":[]} - kaleidx_scope_list = [] - for kaleidx_scope_data in kaleidx_scope: - tmp = kaleidx_scope_data._asdict() + kaleidxscope_list = [] + for kaleidxscope_data in kaleidxscope: + tmp = kaleidxscope_data._asdict() tmp.pop("user") tmp.pop("id") - kaleidx_scope_list.append(tmp) + kaleidxscope_list.append(tmp) return { "userId": data["userId"], - "userKaleidxScopeList": kaleidx_scope_list + "userKaleidxScopeList": kaleidxscope_list } \ No newline at end of file diff --git a/titles/mai2/schema/score.py b/titles/mai2/schema/score.py index f3e7002..d03dba4 100644 --- a/titles/mai2/schema/score.py +++ b/titles/mai2/schema/score.py @@ -175,8 +175,8 @@ playlog_2p = Table( mysql_charset="utf8mb4", ) -kaleidx_scope = Table( - "mai2_score_kaleidx_scope", +kaleidxscope = Table( + "mai2_score_kaleidxscope", metadata, Column("id", Integer, primary_key=True, nullable=False), Column( @@ -481,21 +481,21 @@ class Mai2ScoreData(BaseData): return None return result.scalar() - async def get_user_kaleidx_scope_list(self, user_id: int) -> Optional[List[Row]]: - sql = kaleidx_scope.select(kaleidx_scope.c.user == user_id) + async def get_user_kaleidxscope_list(self, user_id: int) -> Optional[List[Row]]: + sql = kaleidxscope.select(kaleidxscope.c.user == user_id) result = await self.execute(sql) if result is None: return None return result.fetchall() - async def put_user_kaleidx_scope(self, user_id: int, user_kaleidx_scope_data: Dict) -> Optional[int]: - user_kaleidx_scope_data["user"] = user_id - sql = insert(kaleidx_scope).values(**user_kaleidx_scope_data) + async def put_user_kaleidxscope(self, user_id: int, user_kaleidxscope_data: Dict) -> Optional[int]: + user_kaleidxscope_data["user"] = user_id + sql = insert(kaleidxscope).values(**user_kaleidxscope_data) - conflict = sql.on_duplicate_key_update(**user_kaleidx_scope_data) + conflict = sql.on_duplicate_key_update(**user_kaleidxscope_data) result = await self.execute(conflict) if result is None: - self.logger.error(f"put_user_kaleidx_scope: Failed to insert! user_id {user_id}") + self.logger.error(f"put_user_kaleidxscope: Failed to insert! user_id {user_id}") return None return result.lastrowid \ No newline at end of file From 5906bc348677ab0b757494c9418abcdf7a7801be Mon Sep 17 00:00:00 2001 From: daydensteve Date: Sat, 5 Apr 2025 18:05:00 -0400 Subject: [PATCH 107/130] fixed inappropriate use of character illustration id instead of base character id. The Userbox jinja would break if the profile was using an alternate character illustration --- titles/chuni/frontend.py | 2 +- titles/chuni/templates/chuni_userbox.jinja | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titles/chuni/frontend.py b/titles/chuni/frontend.py index d13252c..1059d7c 100644 --- a/titles/chuni/frontend.py +++ b/titles/chuni/frontend.py @@ -457,7 +457,7 @@ class ChuniFrontend(FE_Base): user_characters = [] if not force_unlocked: user_characters = await self.data.item.get_characters(profile.user) - user_characters = [chara["characterId"] for chara in user_characters] + [profile.characterId, profile.charaIllustId] + user_characters = [chara["characterId"] for chara in user_characters] + [profile.characterId] for row in rows: if force_unlocked or row["defaultHave"] or row["characterId"] in user_characters: diff --git a/titles/chuni/templates/chuni_userbox.jinja b/titles/chuni/templates/chuni_userbox.jinja index ab0f821..5114b17 100644 --- a/titles/chuni/templates/chuni_userbox.jinja +++ b/titles/chuni/templates/chuni_userbox.jinja @@ -118,9 +118,9 @@ userbox_components = { "{{ nameplates[profile.nameplateId]["texturePath"] }}", "", "", ""], "character":["{{ characters|length }}", - "{{ profile.charaIllustId }}", - "{{ characters[profile.charaIllustId]["name"] }}", - "{{ characters[profile.charaIllustId]["iconPath"] }}", "", "", ""] + "{{ profile.characterId }}", + "{{ characters[profile.characterId]["name"] }}", + "{{ characters[profile.characterId]["iconPath"] }}", "", "", ""] }; types = Object.keys(userbox_components); orig_trophy = curr_trophy = "{{ profile.trophyId }}"; From eb601e32933ab5b598f6772dff13092da5257744 Mon Sep 17 00:00:00 2001 From: daydensteve Date: Sat, 5 Apr 2025 20:10:07 -0400 Subject: [PATCH 108/130] fixed read failures in older chuni versions where sortname doesn't exist. Also noticed some character import errors associated with & --- titles/chuni/read.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/titles/chuni/read.py b/titles/chuni/read.py index b25d97f..50150e8 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -303,7 +303,7 @@ class ChuniReader(BaseReader): for name in xml_root.findall("name"): id = name.find("id").text name = name.find("str").text - sortName = xml_root.find("sortName").text + sortName = name if xml_root.find("sortName") is None else xml_root.find("sortName").text defaultHave = xml_root.find("defaultHave").text == 'true' disableFlag = xml_root.find("disableFlag") # may not exist in older data is_enabled = True if (disableFlag is None or disableFlag.text == "false") else False @@ -352,12 +352,15 @@ class ChuniReader(BaseReader): if path.exists(f"{root}/{dir}/Chara.xml"): with open(f"{root}/{dir}/Chara.xml", "r", encoding='utf-8') as fp: strdata = fp.read() + # ET may choke if there is a & symbol (which is present in some character xml) + if "&" in strdata: + strdata = strdata.replace("&", "&") xml_root = ET.fromstring(strdata) for name in xml_root.findall("name"): id = name.find("id").text name = name.find("str").text - sortName = xml_root.find("sortName").text + sortName = name if xml_root.find("sortName") is None else xml_root.find("sortName").text for work in xml_root.findall("works"): worksName = work.find("str").text rareType = xml_root.find("rareType").text @@ -401,7 +404,7 @@ class ChuniReader(BaseReader): for name in xml_root.findall("name"): id = name.find("id").text name = name.find("str").text - sortName = xml_root.find("sortName").text + sortName = name if xml_root.find("sortName") is None else xml_root.find("sortName").text for image in xml_root.findall("image"): iconPath = image.find("path").text self.copy_image(iconPath, f"{root}/{dir}", "titles/chuni/img/mapIcon/") @@ -429,7 +432,7 @@ class ChuniReader(BaseReader): for name in xml_root.findall("name"): id = name.find("id").text name = name.find("str").text - sortName = xml_root.find("sortName").text + sortName = name if xml_root.find("sortName") is None else xml_root.find("sortName").text for image in xml_root.findall("image"): imagePath = image.find("path").text self.copy_image(imagePath, f"{root}/{dir}", "titles/chuni/img/systemVoice/") From 1cab68006d21b859aeb21eefe74bab075df5c2f2 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Mon, 7 Apr 2025 18:31:11 -0400 Subject: [PATCH 109/130] add opt static tables --- .../versions/263884e774cc_acc_opt_tables.py | 164 ++++++++++++++++++ core/utils.py | 3 + titles/chuni/const.py | 27 ++- titles/chuni/read.py | 85 ++++++--- titles/chuni/schema/static.py | 133 ++++++++++++-- titles/cm/read.py | 36 ++++ titles/mai2/const.py | 43 +++++ titles/mai2/schema/static.py | 20 ++- titles/ongeki/const.py | 22 ++- titles/ongeki/schema/static.py | 38 +++- 10 files changed, 536 insertions(+), 35 deletions(-) create mode 100644 core/data/alembic/versions/263884e774cc_acc_opt_tables.py diff --git a/core/data/alembic/versions/263884e774cc_acc_opt_tables.py b/core/data/alembic/versions/263884e774cc_acc_opt_tables.py new file mode 100644 index 0000000..1e24813 --- /dev/null +++ b/core/data/alembic/versions/263884e774cc_acc_opt_tables.py @@ -0,0 +1,164 @@ +"""acc_opt_tables + +Revision ID: 263884e774cc +Revises: 1d0014d35220 +Create Date: 2025-04-07 18:05:53.349320 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '263884e774cc' +down_revision = '1d0014d35220' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('chuni_static_opt', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('version', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=4), nullable=False), + sa.Column('sequence', sa.INTEGER(), nullable=False), + sa.Column('whenRead', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.Column('isEnable', sa.BOOLEAN(), server_default='1', nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('version', 'name', name='chuni_static_opt_uk'), + mysql_charset='utf8mb4' + ) + op.create_table('cm_static_opts', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('version', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=4), nullable=False), + sa.Column('sequence', sa.INTEGER(), nullable=True), + sa.Column('gekiVersion', sa.INTEGER(), nullable=True), + sa.Column('gekiReleaseVer', sa.INTEGER(), nullable=True), + sa.Column('maiVersion', sa.INTEGER(), nullable=True), + sa.Column('maiReleaseVer', sa.INTEGER(), nullable=True), + sa.Column('whenRead', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.Column('isEnable', sa.BOOLEAN(), server_default='1', nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('version', 'name', name='cm_static_opts_uk'), + mysql_charset='utf8mb4' + ) + op.create_table('mai2_static_opt', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('version', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=4), nullable=False), + sa.Column('sequence', sa.INTEGER(), nullable=False), + sa.Column('cmReleaseVer', sa.INTEGER(), nullable=False), + sa.Column('whenRead', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.Column('isEnable', sa.BOOLEAN(), server_default='1', nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('version', 'name', name='mai2_static_opt_uk'), + mysql_charset='utf8mb4' + ) + op.create_table('ongeki_static_opt', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('version', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=4), nullable=False), + sa.Column('sequence', sa.INTEGER(), nullable=False), + sa.Column('cmReleaseVer', sa.INTEGER(), nullable=False), + sa.Column('whenRead', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.Column('isEnable', sa.BOOLEAN(), server_default='1', nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('version', 'name', name='ongeki_static_opt_uk'), + mysql_charset='utf8mb4' + ) + op.add_column('chuni_static_avatar', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_avatar', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_cards', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_cards', 'cm_static_opts', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_character', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_character', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_charge', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_charge', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_events', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_events', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_gachas', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_gachas', 'cm_static_opts', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_login_bonus', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_login_bonus', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_login_bonus_preset', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_login_bonus_preset', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_map_icon', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_map_icon', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_music', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_music', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_system_voice', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_system_voice', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('chuni_static_trophy', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_trophy', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('mai2_static_cards', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'mai2_static_cards', 'cm_static_opts', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('mai2_static_event', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'mai2_static_event', 'mai2_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('mai2_static_music', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'mai2_static_music', 'mai2_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('mai2_static_ticket', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'mai2_static_ticket', 'mai2_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('ongeki_static_cards', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'ongeki_static_cards', 'ongeki_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('ongeki_static_events', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'ongeki_static_events', 'ongeki_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('ongeki_static_gachas', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'ongeki_static_gachas', 'cm_static_opts', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('ongeki_static_music', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'ongeki_static_music', 'ongeki_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + op.add_column('ongeki_static_rewards', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'ongeki_static_rewards', 'ongeki_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("ongeki_static_rewards_ibfk_1", 'ongeki_static_rewards', type_='foreignkey') + op.drop_column('ongeki_static_rewards', 'opt') + op.drop_constraint("ongeki_static_music_ibfk_1", 'ongeki_static_music', type_='foreignkey') + op.drop_column('ongeki_static_music', 'opt') + op.drop_constraint("ongeki_static_gachas_ibfk_1", 'ongeki_static_gachas', type_='foreignkey') + op.drop_column('ongeki_static_gachas', 'opt') + op.drop_constraint("ongeki_static_events_ibfk_1", "ongeki_static_events", type_='foreignkey') + op.drop_column('ongeki_static_events', 'opt') + op.drop_constraint("ongeki_static_cards_ibfk_1", "ongeki_static_cards", type_='foreignkey') + op.drop_column('ongeki_static_cards', 'opt') + op.drop_constraint("mai2_static_ticket_ibfk_1", "mai2_static_ticket", type_='foreignkey') + op.drop_column('mai2_static_ticket', 'opt') + op.drop_constraint("mai2_static_music_ibfk_1", "mai2_static_music", type_='foreignkey') + op.drop_column('mai2_static_music', 'opt') + op.drop_constraint("mai2_static_event_ibfk_1", "mai2_static_event", type_='foreignkey') + op.drop_column('mai2_static_event', 'opt') + op.drop_constraint("mai2_static_cards_ibfk_1", "mai2_static_cards", type_='foreignkey') + op.drop_column('mai2_static_cards', 'opt') + op.drop_constraint("chuni_static_trophy_ibfk_1", "chuni_static_trophy", type_='foreignkey') + op.drop_column('chuni_static_trophy', 'opt') + op.drop_constraint("chuni_static_system_voice_ibfk_1", "chuni_static_system_voice", type_='foreignkey') + op.drop_column('chuni_static_system_voice', 'opt') + op.drop_constraint("chuni_static_music_ibfk_1", "chuni_static_music", type_='foreignkey') + op.drop_column('chuni_static_music', 'opt') + op.drop_constraint("chuni_static_map_icon_ibfk_1", "chuni_static_map_icon", type_='foreignkey') + op.drop_column('chuni_static_map_icon', 'opt') + op.drop_constraint("chuni_static_login_bonus_preset_ibfk_1", "chuni_static_login_bonus_preset", type_='foreignkey') + op.drop_column('chuni_static_login_bonus_preset', 'opt') + op.drop_constraint("chuni_static_login_bonus_ibfk_2", "chuni_static_login_bonus", type_='foreignkey') + op.drop_column('chuni_static_login_bonus', 'opt') + op.drop_constraint("chuni_static_gachas_ibfk_1", "chuni_static_gachas", type_='foreignkey') + op.drop_column('chuni_static_gachas', 'opt') + op.drop_constraint("chuni_static_events_ibfk_1", "chuni_static_events", type_='foreignkey') + op.drop_column('chuni_static_events', 'opt') + op.drop_constraint("chuni_static_charge_ibfk_1", "chuni_static_charge", type_='foreignkey') + op.drop_column('chuni_static_charge', 'opt') + op.drop_constraint("chuni_static_character_ibfk_1", "chuni_static_character", type_='foreignkey') + op.drop_column('chuni_static_character', 'opt') + op.drop_constraint("chuni_static_cards_ibfk_1", "chuni_static_cards", type_='foreignkey') + op.drop_column('chuni_static_cards', 'opt') + op.drop_constraint("chuni_static_avatar_ibfk_1", "chuni_static_avatar", type_='foreignkey') + op.drop_column('chuni_static_avatar', 'opt') + op.drop_table('ongeki_static_opt') + op.drop_table('mai2_static_opt') + op.drop_table('cm_static_opts') + op.drop_table('chuni_static_opt') + # ### end Alembic commands ### diff --git a/core/utils.py b/core/utils.py index af96451..92f9bf5 100644 --- a/core/utils.py +++ b/core/utils.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from os import walk from types import ModuleType from typing import Any, Dict, Optional +import math import jwt from starlette.requests import Request @@ -92,6 +93,8 @@ class Utils: return cls.real_title_port_ssl +def floor_to_nearest_005(version: int) -> int: + return (version // 5) * 5 def create_sega_auth_key( aime_id: int, diff --git a/titles/chuni/const.py b/titles/chuni/const.py index d0d73d5..fd05003 100644 --- a/titles/chuni/const.py +++ b/titles/chuni/const.py @@ -1,5 +1,6 @@ from enum import Enum, IntEnum - +from typing import Optional +from core.utils import floor_to_nearest_005 class ChuniConstants: GAME_CODE = "SDBT" @@ -78,10 +79,34 @@ class ChuniConstants: ( 0, "D"), ] + VERSION_LUT = { + "100": VER_CHUNITHM, + "105": VER_CHUNITHM_PLUS, + "110": VER_CHUNITHM_AIR, + "115": VER_CHUNITHM_AIR_PLUS, + "120": VER_CHUNITHM_STAR, + "125": VER_CHUNITHM_STAR_PLUS, + "130": VER_CHUNITHM_AMAZON, + "135": VER_CHUNITHM_AMAZON_PLUS, + "140": VER_CHUNITHM_CRYSTAL, + "145": VER_CHUNITHM_CRYSTAL_PLUS, + "150": VER_CHUNITHM_PARADISE, + "200": VER_CHUNITHM_NEW, + "205": VER_CHUNITHM_NEW_PLUS, + "210": VER_CHUNITHM_SUN, + "215": VER_CHUNITHM_SUN_PLUS, + "220": VER_CHUNITHM_LUMINOUS, + "225": VER_CHUNITHM_LUMINOUS_PLUS, + } + @classmethod def game_ver_to_string(cls, ver: int): return cls.VERSION_NAMES[ver] + @classmethod + def int_ver_to_game_ver(cls, ver: int) -> Optional[int]: + """ Takes an int ver (ex 100 for 1.00) and returns an internal game version """ + return cls.VERSION_LUT.get(str(floor_to_nearest_005(ver)), None) class MapAreaConditionType(IntEnum): """Condition types for the GetGameMapAreaConditionApi endpoint. Incomplete. diff --git a/titles/chuni/read.py b/titles/chuni/read.py index b25d97f..fe0c411 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -3,6 +3,7 @@ from os import walk, path import xml.etree.ElementTree as ET from read import BaseReader from PIL import Image +import configparser from core.config import CoreConfig from titles.chuni.database import ChuniData @@ -50,18 +51,19 @@ class ChuniReader(BaseReader): for dir in data_dirs: self.logger.info(f"Read from {dir}") - await self.read_events(f"{dir}/event") - await self.read_music(f"{dir}/music", we_diff) - await self.read_charges(f"{dir}/chargeItem") - await self.read_avatar(f"{dir}/avatarAccessory") - await self.read_login_bonus(f"{dir}/") - await self.read_nameplate(f"{dir}/namePlate") - await self.read_trophy(f"{dir}/trophy") - await self.read_character(f"{dir}/chara", dds_images) - await self.read_map_icon(f"{dir}/mapIcon") - await self.read_system_voice(f"{dir}/systemVoice") + this_opt_id = await self.read_opt_info(dir) # this also treats A000 as an opt, which is intended + await self.read_events(f"{dir}/event", this_opt_id) + await self.read_music(f"{dir}/music", we_diff, this_opt_id) + await self.read_charges(f"{dir}/chargeItem", this_opt_id) + await self.read_avatar(f"{dir}/avatarAccessory", this_opt_id) + await self.read_login_bonus(f"{dir}/", this_opt_id) + await self.read_nameplate(f"{dir}/namePlate", this_opt_id) + await self.read_trophy(f"{dir}/trophy", this_opt_id) + await self.read_character(f"{dir}/chara", dds_images, this_opt_id) + await self.read_map_icon(f"{dir}/mapIcon", this_opt_id) + await self.read_system_voice(f"{dir}/systemVoice", this_opt_id) - async def read_login_bonus(self, root_dir: str) -> None: + async def read_login_bonus(self, root_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(f"{root_dir}loginBonusPreset"): for dir in dirs: if path.exists(f"{root}/{dir}/LoginBonusPreset.xml"): @@ -132,7 +134,7 @@ class ChuniReader(BaseReader): f"Failed to insert login bonus {bonus_id}" ) - async def read_events(self, evt_dir: str) -> None: + async def read_events(self, evt_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(evt_dir): for dir in dirs: if path.exists(f"{root}/{dir}/Event.xml"): @@ -154,7 +156,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert event {id}") - async def read_music(self, music_dir: str, we_diff: str = "4") -> None: + async def read_music(self, music_dir: str, we_diff: str = "4", opt_id: Optional[int] = None) -> None: max_title_len = MusicTable.columns["title"].type.length max_artist_len = MusicTable.columns["artist"].type.length @@ -230,7 +232,7 @@ class ChuniReader(BaseReader): f"Failed to insert music {song_id} chart {chart_id}" ) - async def read_charges(self, charge_dir: str) -> None: + async def read_charges(self, charge_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(charge_dir): for dir in dirs: if path.exists(f"{root}/{dir}/ChargeItem.xml"): @@ -259,7 +261,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert charge {id}") - async def read_avatar(self, avatar_dir: str) -> None: + async def read_avatar(self, avatar_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(avatar_dir): for dir in dirs: if path.exists(f"{root}/{dir}/AvatarAccessory.xml"): @@ -292,7 +294,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert avatarAccessory {id}") - async def read_nameplate(self, nameplate_dir: str) -> None: + async def read_nameplate(self, nameplate_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(nameplate_dir): for dir in dirs: if path.exists(f"{root}/{dir}/NamePlate.xml"): @@ -321,7 +323,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert nameplate {id}") - async def read_trophy(self, trophy_dir: str) -> None: + async def read_trophy(self, trophy_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(trophy_dir): for dir in dirs: if path.exists(f"{root}/{dir}/Trophy.xml"): @@ -346,7 +348,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert trophy {id}") - async def read_character(self, chara_dir: str, dds_images: dict) -> None: + async def read_character(self, chara_dir: str, dds_images: dict, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(chara_dir): for dir in dirs: if path.exists(f"{root}/{dir}/Chara.xml"): @@ -390,7 +392,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to insert character {id}") - async def read_map_icon(self, mapicon_dir: str) -> None: + async def read_map_icon(self, mapicon_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(mapicon_dir): for dir in dirs: if path.exists(f"{root}/{dir}/MapIcon.xml"): @@ -418,7 +420,7 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to map icon {id}") - async def read_system_voice(self, voice_dir: str) -> None: + async def read_system_voice(self, voice_dir: str, opt_id: Optional[int] = None) -> None: for root, dirs, files in walk(voice_dir): for dir in dirs: if path.exists(f"{root}/{dir}/SystemVoice.xml"): @@ -446,6 +448,49 @@ class ChuniReader(BaseReader): else: self.logger.warning(f"Failed to system voice {id}") + async def read_opt_info(self, directory: str) -> Optional[int]: + if not path.exists(f"{directory}/data.conf"): + self.logger.warning(f"{directory} does not contain data.conf, opt info will not be read") + return None + + data_config = configparser.ConfigParser() + if not data_config.read(f"{directory}/data.conf", 'utf-8'): + self.logger.warning(f"{directory}/data.conf failed to read or parse, opt info will not be read") + return None + + if 'Version' not in data_config: + self.logger.warning(f"{directory}/data.conf contains no Version section, opt info will not be read") + return None + + if 'Name' not in data_config['Version']: # Probably not worth checking that the other sections exist + self.logger.warning(f"{directory}/data.conf contains no Name item in the Version section, opt info will not be read") + return None + + if 'VerMajor' not in data_config['Version']: # Probably not worth checking that the other sections exist + self.logger.warning(f"{directory}/data.conf contains no VerMajor item in the Version section, opt info will not be read") + return None + + if 'VerMinor' not in data_config['Version']: # Probably not worth checking that the other sections exist + self.logger.warning(f"{directory}/data.conf contains no VerMinor item in the Version section, opt info will not be read") + return None + + if 'VerRelease' not in data_config['Version']: # Probably not worth checking that the other sections exist + self.logger.warning(f"{directory}/data.conf contains no VerRelease item in the Version section, opt info will not be read") + return None + + opt_seq = data_config['Version']['VerRelease'] + opt_folder = path.basename(path.normpath(directory)) + opt_id = await self.data.static.get_opt_by_version_folder(self.version, opt_folder) + + if not opt_id: + opt_id = await self.data.static.put_opt(self.version, opt_folder, opt_seq) + if not opt_id: + self.logger.error(f"Failed to put opt folder info for {opt_folder}") + return None + + self.logger.info(f"Opt folder {opt_folder} (Database ID {opt_id}) contains {data_config['Version']['Name']} v{data_config['Version']['VerMajor']}.{data_config['Version']['VerMinor']}.{opt_seq}") + return opt_id + def copy_image(self, filename: str, src_dir: str, dst_dir: str) -> None: # Convert the image to png so we can easily display it in the frontend file_src = path.join(src_dir, filename) diff --git a/titles/chuni/schema/static.py b/titles/chuni/schema/static.py index e3070ec..0f7dc4a 100644 --- a/titles/chuni/schema/static.py +++ b/titles/chuni/schema/static.py @@ -7,8 +7,7 @@ from sqlalchemy import ( PrimaryKeyConstraint, and_, ) -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float -from sqlalchemy.engine.base import Connection +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, BIGINT, Float, INTEGER, VARCHAR, BOOLEAN from sqlalchemy.engine import Row from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select @@ -17,6 +16,19 @@ from datetime import datetime from core.data.schema import BaseData, metadata +opts = Table( + "chuni_static_opt", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column("version", INTEGER, nullable=False), + Column("name", VARCHAR(4), nullable=False), # Axxx + Column("sequence", INTEGER, nullable=False), # VerRelease in data.conf + Column("whenRead", TIMESTAMP, nullable=False, server_default=func.now()), + Column("isEnable", BOOLEAN, nullable=False, server_default="1"), + UniqueConstraint("version", "name", name="chuni_static_opt_uk"), + mysql_charset="utf8mb4", +) + events = Table( "chuni_static_events", metadata, @@ -27,6 +39,7 @@ events = Table( Column("name", String(255)), Column("startDate", TIMESTAMP, server_default=func.now()), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "eventId", name="chuni_static_events_uk"), mysql_charset="utf8mb4", ) @@ -44,6 +57,7 @@ music = Table( Column("genre", String(255)), Column("jacketPath", String(255)), Column("worldsEndTag", String(7)), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "songId", "chartId", name="chuni_static_music_uk"), mysql_charset="utf8mb4", ) @@ -59,6 +73,7 @@ charge = Table( Column("consumeType", Integer), Column("sellingAppeal", Boolean), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "chargeId", name="chuni_static_charge_uk"), mysql_charset="utf8mb4", ) @@ -76,6 +91,7 @@ avatar = Table( Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), Column("sortName", String(255)), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "avatarAccessoryId", name="chuni_static_avatar_uk"), mysql_charset="utf8mb4", ) @@ -110,6 +126,7 @@ character = Table( Column("imagePath3", String(255)), Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "characterId", name="chuni_static_character_uk"), mysql_charset="utf8mb4", ) @@ -124,6 +141,7 @@ trophy = Table( Column("rareType", Integer), Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "trophyId", name="chuni_static_trophy_uk"), mysql_charset="utf8mb4", ) @@ -139,6 +157,7 @@ map_icon = Table( Column("iconPath", String(255)), Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "mapIconId", name="chuni_static_mapicon_uk"), mysql_charset="utf8mb4", ) @@ -154,6 +173,7 @@ system_voice = Table( Column("imagePath", String(255)), Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "voiceId", name="chuni_static_systemvoice_uk"), mysql_charset="utf8mb4", ) @@ -175,6 +195,7 @@ gachas = Table( Column("endDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"), Column("noticeStartDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"), Column("noticeEndDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"), + Column("opt", ForeignKey("cm_static_opts.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "gachaId", "gachaName", name="chuni_static_gachas_uk"), mysql_charset="utf8mb4", ) @@ -195,6 +216,7 @@ cards = Table( Column("combo", Integer, nullable=False), Column("chain", Integer, nullable=False), Column("skillName", String(255), nullable=False), + Column("opt", ForeignKey("cm_static_opts.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "cardId", name="chuni_static_cards_uk"), mysql_charset="utf8mb4", ) @@ -219,6 +241,7 @@ login_bonus_preset = Table( Column("version", Integer, nullable=False), Column("presetName", String(255), nullable=False), Column("isEnabled", Boolean, server_default="1"), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), PrimaryKeyConstraint( "presetId", "version", name="chuni_static_login_bonus_preset_pk" ), @@ -238,6 +261,7 @@ login_bonus = Table( Column("itemNum", Integer, nullable=False), Column("needLoginDayCount", Integer, nullable=False), Column("loginBonusCategoryType", Integer, nullable=False), + Column("opt", BIGINT), UniqueConstraint( "version", "presetId", "loginBonusId", name="chuni_static_login_bonus_uk" ), @@ -251,10 +275,18 @@ login_bonus = Table( ondelete="CASCADE", name="chuni_static_login_bonus_ibfk_1", ), + ForeignKeyConstraint( + ["opt"], + [ + "chuni_static_opt.id", + ], + onupdate="SET NULL", + ondelete="CASCADE", + name="chuni_static_login_bonus_ibfk_2", + ), mysql_charset="utf8mb4", ) - class ChuniStaticData(BaseData): async def put_login_bonus( self, @@ -327,17 +359,17 @@ class ChuniStaticData(BaseData): return result.fetchone() async def put_login_bonus_preset( - self, version: int, preset_id: int, preset_name: str, is_enabled: bool + self, version: int, preset_id: int, preset_name: str, isEnabled: bool ) -> Optional[int]: sql = insert(login_bonus_preset).values( presetId=preset_id, version=version, presetName=preset_name, - isEnabled=is_enabled, + isEnabled=isEnabled, ) conflict = sql.on_duplicate_key_update( - presetName=preset_name, isEnabled=is_enabled + presetName=preset_name, isEnabled=isEnabled ) result = await self.execute(conflict) @@ -346,12 +378,12 @@ class ChuniStaticData(BaseData): return result.lastrowid async def get_login_bonus_presets( - self, version: int, is_enabled: bool = True + self, version: int, isEnabled: bool = True ) -> Optional[List[Row]]: sql = login_bonus_preset.select( and_( login_bonus_preset.c.version == version, - login_bonus_preset.c.isEnabled == is_enabled, + login_bonus_preset.c.isEnabled == isEnabled, ) ) @@ -542,7 +574,6 @@ class ChuniStaticData(BaseData): return None return result.fetchone() - async def put_avatar( self, version: int, @@ -926,4 +957,86 @@ class ChuniStaticData(BaseData): result = await self.execute(sql) if result is None: return None - return result.fetchone() \ No newline at end of file + return result.fetchone() + + async def put_opt(self, version: int, folder: str, sequence: int) -> Optional[int]: + sql = insert(opts).values(version=version, name=folder, sequence=sequence) + + conflict = sql.on_duplicate_key_update(sequence=sequence, whenRead=datetime.now()) + + result = await self.execute(conflict) + if result is None: + self.logger.warning(f"Failed to insert opt! version {version} folder {folder} sequence {sequence}") + return None + return result.lastrowid + + async def get_opt_by_version_folder(self, version: int, folder: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.name == folder, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opt_by_version_sequence(self, version: int, sequence: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.sequence == sequence, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opts_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(opts.c.version == version)) + + if result is None: + return None + return result.fetchall() + + async def get_opts_enabled_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + ))) + + if result is None: + return None + return result.fetchall() + + async def get_latest_enabled_opt_by_version(self, version: int) -> Optional[Row]: + result = await self.execute( + opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + )).order_by(opts.c.sequence.desc()) + ) + + if result is None: + return None + return result.fetchone() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def set_opt_enabled(self, opt_id: int, enabled: bool) -> bool: + result = await self.execute(opts.update(opts.c.id == opt_id).values(isEnable=enabled)) + + if result is None: + self.logger.error(f"Failed to set opt enabled status to {enabled} for opt {opt_id}") + return False + return True diff --git a/titles/cm/read.py b/titles/cm/read.py index b4b3b5e..d0db43c 100644 --- a/titles/cm/read.py +++ b/titles/cm/read.py @@ -325,3 +325,39 @@ class CardMakerReader(BaseReader): maxSelectPoint=max_select_point, ) self.logger.info(f"Added ongeki gacha {gacha_id}") + + async def read_opt(self, base_dir: str) -> None: + self.logger.info(f"Reading opt data from {base_dir}...") + cm_data_cfg = None + cm_data_cfg_file = os.path.join(base_dir, "DataConfig.xml") + + geki_data_cfg = None + geki_data_cfg_file = os.path.join(base_dir, "GEKI", "DataConfig.xml") + + mai2_data_cfg = None + mai2_data_cfg_file = os.path.join(base_dir, "MAI", "DataConfig.xml") + + if os.path.exists(cm_data_cfg_file): + with open(cm_data_cfg_file, "r") as f: + cm_data_cfg = ET.fromstring(f.read()) + else: + self.logger.info(f"No DataConfig.xml in {base_dir}, sequence will be null") + + if os.path.exists(geki_data_cfg_file): + with open(geki_data_cfg_file, "r") as f: + geki_data_cfg = ET.fromstring(f.read()) + else: + self.logger.info(f"Cannot find {geki_data_cfg_file}, gekiVersion and gekiReleaseVer will be null") + + if os.path.exists(mai2_data_cfg_file): + with open(mai2_data_cfg_file, "r") as f: + mai2_data_cfg = ET.fromstring(f.read()) + else: + self.logger.info(f"Cannot find {mai2_data_cfg_file}, mai2Version and mai2ReleaseVer will be null") + + cm_rel_ver = int(cm_data_cfg.find("DataConfig/version/release").text) + + geki_rel_ver = int(geki_data_cfg.find("DataConfig/version/release").text) + + mai2_rel_ver = int(mai2_data_cfg.find("DataConfig/version/release").text) + mai2_db_ver = Mai2Constants.int_ver_to_game_ver(mai2_data_cfg.find("DataConfig/version/major").text + mai2_data_cfg.find("DataConfig/version/minor").text) diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 0d13a0d..68d3e80 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -1,3 +1,6 @@ +from typing import Optional +from core.utils import floor_to_nearest_005 + class Mai2Constants: GRADE = { "D": 0, @@ -83,6 +86,46 @@ class Mai2Constants: "maimai DX BUDDiES PLUS" ) + MAI_VERSION_LUT = { + "100": VER_MAIMAI, + "110": VER_MAIMAI_PLUS, + "120": VER_MAIMAI_GREEN, + "130": VER_MAIMAI_GREEN_PLUS, + "140": VER_MAIMAI_ORANGE, + "150": VER_MAIMAI_ORANGE_PLUS, + "160": VER_MAIMAI_PINK, + "170": VER_MAIMAI_PINK_PLUS, + "180": VER_MAIMAI_MURASAKI, + "185": VER_MAIMAI_MURASAKI_PLUS, + "190": VER_MAIMAI_MILK, + "195": VER_MAIMAI_MILK_PLUS, + "197": VER_MAIMAI_FINALE, + } + + MAI2_VERSION_LUT = { + "100": VER_MAIMAI_DX, + "105": VER_MAIMAI_DX_PLUS, + "110": VER_MAIMAI_DX_SPLASH, + "115": VER_MAIMAI_DX_SPLASH_PLUS, + "120": VER_MAIMAI_DX_UNIVERSE, + "125": VER_MAIMAI_DX_UNIVERSE_PLUS, + "130": VER_MAIMAI_DX_FESTIVAL, + "135": VER_MAIMAI_DX_FESTIVAL_PLUS, + "140": VER_MAIMAI_DX_BUDDIES, + "145": VER_MAIMAI_DX_BUDDIES_PLUS, + } + @classmethod def game_ver_to_string(cls, ver: int): + """ Takes an internal game version (ex 13 for maimai DX) and returns a the full name of the version """ return cls.VERSION_STRING[ver] + + @classmethod + def int_ver_to_game_ver(cls, ver: int, is_dx = True) -> Optional[int]: + """ Takes an int ver (ex 100 for 1.00) and returns an internal game version """ + if is_dx: + return cls.MAI2_VERSION_LUT.get(str(floor_to_nearest_005(ver)), None) + else: + if ver >= 197: + return cls.VER_MAIMAI_FINALE + return cls.MAI_VERSION_LUT.get(str(floor_to_nearest_005(ver)), None) diff --git a/titles/mai2/schema/static.py b/titles/mai2/schema/static.py index ddba0f8..33b93c6 100644 --- a/titles/mai2/schema/static.py +++ b/titles/mai2/schema/static.py @@ -2,13 +2,27 @@ from core.data.schema.base import BaseData, metadata from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, BIGINT, Float, INTEGER, BOOLEAN, VARCHAR from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row from sqlalchemy.dialects.mysql import insert from datetime import datetime +opts = Table( + "mai2_static_opt", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column("version", INTEGER, nullable=False), + Column("name", VARCHAR(4), nullable=False), # Axxx + Column("sequence", INTEGER, nullable=False), # release in DataConfig.xml + Column("cmReleaseVer", INTEGER, nullable=False), + Column("whenRead", TIMESTAMP, nullable=False, server_default=func.now()), + Column("isEnable", BOOLEAN, nullable=False, server_default="1"), + UniqueConstraint("version", "name", name="mai2_static_opt_uk"), + mysql_charset="utf8mb4", +) + event = Table( "mai2_static_event", metadata, @@ -19,6 +33,7 @@ event = Table( Column("name", String(255)), Column("startDate", TIMESTAMP, server_default=func.now()), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("mai2_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "eventId", "type", name="mai2_static_event_uk"), mysql_charset="utf8mb4", ) @@ -37,6 +52,7 @@ music = Table( Column("addedVersion", String(255)), Column("difficulty", Float), Column("noteDesigner", String(255)), + Column("opt", ForeignKey("mai2_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("songId", "chartId", "version", name="mai2_static_music_uk"), mysql_charset="utf8mb4", ) @@ -51,6 +67,7 @@ ticket = Table( Column("name", String(255)), Column("price", Integer, server_default="1"), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("mai2_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "ticketId", name="mai2_static_ticket_uk"), mysql_charset="utf8mb4", ) @@ -67,6 +84,7 @@ cards = Table( Column("noticeStartDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"), Column("noticeEndDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("cm_static_opts.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "cardId", "cardName", name="mai2_static_cards_uk"), mysql_charset="utf8mb4", ) diff --git a/titles/ongeki/const.py b/titles/ongeki/const.py index 71ea7f2..f218fcc 100644 --- a/titles/ongeki/const.py +++ b/titles/ongeki/const.py @@ -1,6 +1,6 @@ -from typing import Final, Dict +from typing import Optional from enum import Enum - +from core.utils import floor_to_nearest_005 class OngekiConstants: GAME_CODE = "SDDT" @@ -106,6 +106,24 @@ class OngekiConstants: "O.N.G.E.K.I. bright MEMORY Act.3", ) + VERSION_LUT = { + "100": VER_ONGEKI, + "105": VER_ONGEKI_PLUS, + "110": VER_ONGEKI_SUMMER, + "115": VER_ONGEKI_SUMMER_PLUS, + "120": VER_ONGEKI_RED, + "125": VER_ONGEKI_RED_PLUS, + "130": VER_ONGEKI_BRIGHT, + "135": VER_ONGEKI_BRIGHT_MEMORY, + "140": VER_ONGEKI_BRIGHT_MEMORY, + "145": VER_ONGEKI_BRIGHT_MEMORY_ACT3, + } + @classmethod def game_ver_to_string(cls, ver: int): return cls.VERSION_NAMES[ver] + + @classmethod + def int_ver_to_game_ver(cls, ver: int) -> Optional[int]: + """ Takes an int ver (ex 100 for 1.00) and returns an internal game version """ + return cls.VERSION_LUT.get(str(floor_to_nearest_005(ver)), None) diff --git a/titles/ongeki/schema/static.py b/titles/ongeki/schema/static.py index 5d6a3e0..8609f5c 100644 --- a/titles/ongeki/schema/static.py +++ b/titles/ongeki/schema/static.py @@ -1,6 +1,6 @@ from typing import Dict, List, Optional from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, BIGINT, Float, INTEGER, VARCHAR, BOOLEAN from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -9,6 +9,37 @@ from sqlalchemy.dialects.mysql import insert from core.data.schema import BaseData, metadata from core.data.schema.arcade import machine +opts = Table( + "ongeki_static_opt", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column("version", INTEGER, nullable=False), + Column("name", VARCHAR(4), nullable=False), # Axxx + Column("sequence", INTEGER, nullable=False), # release in DataConfig.xml + Column("cmReleaseVer", INTEGER, nullable=False), + Column("whenRead", TIMESTAMP, nullable=False, server_default=func.now()), + Column("isEnable", BOOLEAN, nullable=False, server_default="1"), + UniqueConstraint("version", "name", name="ongeki_static_opt_uk"), + mysql_charset="utf8mb4", +) + +cm_opts = Table( + "cm_static_opts", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column("version", INTEGER, nullable=False), + Column("name", VARCHAR(4), nullable=False), # Axxx + Column("sequence", INTEGER), # Not all opts have a DataConfig.xml + Column("gekiVersion", INTEGER), # GEKI/DataConfig.xml + Column("gekiReleaseVer", INTEGER), # GEKI/DataConfig.xml + Column("maiVersion", INTEGER), # MAI/DataConfig.xml + Column("maiReleaseVer", INTEGER), # MAI/DataConfig.xml + Column("whenRead", TIMESTAMP, nullable=False, server_default=func.now()), + Column("isEnable", BOOLEAN, nullable=False, server_default="1"), + UniqueConstraint("version", "name", name="cm_static_opts_uk"), + mysql_charset="utf8mb4", +) + events = Table( "ongeki_static_events", metadata, @@ -20,6 +51,7 @@ events = Table( Column("startDate", TIMESTAMP, server_default=func.now()), Column("endDate", TIMESTAMP, server_default=func.now()), Column("enabled", Boolean, server_default="1"), + Column("opt", ForeignKey("ongeki_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "eventId", "type", name="ongeki_static_events_uk"), mysql_charset="utf8mb4", ) @@ -36,6 +68,7 @@ music = Table( Column("artist", String(255)), Column("genre", String(255)), Column("level", Float), + Column("opt", ForeignKey("ongeki_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "songId", "chartId", name="ongeki_static_music_uk"), mysql_charset="utf8mb4", ) @@ -59,6 +92,7 @@ gachas = Table( Column("noticeStartDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"), Column("noticeEndDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"), Column("convertEndDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"), + Column("opt", ForeignKey("cm_static_opts.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "gachaId", "gachaName", name="ongeki_static_gachas_uk"), mysql_charset="utf8mb4", ) @@ -94,6 +128,7 @@ cards = Table( Column("skillId", Integer, nullable=False), Column("choKaikaSkillId", Integer, nullable=False), Column("cardNumber", String(255)), + Column("opt", ForeignKey("ongeki_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "cardId", name="ongeki_static_cards_uk"), mysql_charset="utf8mb4", ) @@ -107,6 +142,7 @@ rewards = Table( Column("rewardname", String(255), nullable=False), Column("itemKind", Integer, nullable=False), Column("itemId", Integer, nullable=False), + Column("opt", ForeignKey("ongeki_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "rewardId", name="ongeki_static_rewards_uk"), mysql_charset="utf8mb4", ) From ed2b6044ff024274d0f33a99055aafaf4fbebb40 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Tue, 8 Apr 2025 11:18:40 +0800 Subject: [PATCH 110/130] mai2_item_present fixed --- titles/mai2/schema/item.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titles/mai2/schema/item.py b/titles/mai2/schema/item.py index 3b7d8d4..8639ae5 100644 --- a/titles/mai2/schema/item.py +++ b/titles/mai2/schema/item.py @@ -728,10 +728,11 @@ class Mai2ItemData(BaseData): # Do an anti-join with the mai2_item_item table to exclude any # items the users have already owned. if exclude_owned: - sql = sql.join( + sql = sql.outerjoin( item, (present.c.itemKind == item.c.itemKind) & (present.c.itemId == item.c.itemId) + & (item.c.user == user_id) ) condition &= (item.c.itemKind.is_(None) & item.c.itemId.is_(None)) From 703068e9659d15ab2c40d17c26042a9cd378eea8 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Tue, 8 Apr 2025 08:11:01 +0800 Subject: [PATCH 111/130] delete unused alembic file create new alembic file --- ....py => 5cf98cfe52ad_mai2_prism_support.py} | 18 ++++++------ .../d0f1c7fa9505_mai2_add_prism_support.py | 28 ------------------- docs/game_specific_info.md | 8 +++--- 3 files changed, 14 insertions(+), 40 deletions(-) rename core/data/alembic/versions/{16f34bf7b968_mai2_kaleidx_scope_support.py => 5cf98cfe52ad_mai2_prism_support.py} (79%) delete mode 100644 core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py diff --git a/core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py b/core/data/alembic/versions/5cf98cfe52ad_mai2_prism_support.py similarity index 79% rename from core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py rename to core/data/alembic/versions/5cf98cfe52ad_mai2_prism_support.py index b8baa1a..77ca08a 100644 --- a/core/data/alembic/versions/16f34bf7b968_mai2_kaleidx_scope_support.py +++ b/core/data/alembic/versions/5cf98cfe52ad_mai2_prism_support.py @@ -1,8 +1,8 @@ -"""Mai2 Kaleidx Scope Support +"""Mai2 PRiSM support -Revision ID: 16f34bf7b968 -Revises: d0f1c7fa9505 -Create Date: 2025-04-02 07:06:15.829591 +Revision ID: 5cf98cfe52ad +Revises: 263884e774cc +Create Date: 2025-04-08 08:00:51.243089 """ from alembic import op @@ -10,15 +10,15 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '16f34bf7b968' -down_revision = 'd0f1c7fa9505' +revision = '5cf98cfe52ad' +down_revision = '263884e774cc' branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('mai2_score_kaleidx_scope', + op.create_table('mai2_score_kaleidxscope', sa.Column('id', sa.Integer(), nullable=False), sa.Column('user', sa.Integer(), nullable=False), sa.Column('gateId', sa.Integer(), nullable=True), @@ -41,10 +41,12 @@ def upgrade(): sa.UniqueConstraint('user', 'gateId', name='mai2_score_best_uk'), mysql_charset='utf8mb4' ) + op.add_column('mai2_playlog', sa.Column('extBool2', sa.Boolean(), nullable=True, server_default=sa.text("NULL"))) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('mai2_score_kaleidx_scope') + op.drop_column('mai2_playlog', 'extBool2') + op.drop_table('mai2_score_kaleidxscope') # ### end Alembic commands ### diff --git a/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py b/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py deleted file mode 100644 index c879706..0000000 --- a/core/data/alembic/versions/d0f1c7fa9505_mai2_add_prism_support.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Mai2 add PRiSM support - -Revision ID: d0f1c7fa9505 -Revises: 1d0014d35220 -Create Date: 2025-04-02 06:37:10.657372 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'd0f1c7fa9505' -down_revision = '1d0014d35220' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('mai2_playlog', sa.Column('extBool2', sa.Boolean(), nullable=True,server_default=sa.text("NULL"))) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('mai2_playlog', 'extBool2') - # ### end Alembic commands ### diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 6bb3f67..7121478 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -195,10 +195,10 @@ Config file is located in `config/cxb.yaml`. ### Presents Presents are items given to the user when they login, with a little animation (for example, the KOP song was given to the finalists as a present). To add a present, you must insert it into the `mai2_item_present` table. In that table, a NULL version means any version, a NULL user means any user, a NULL start date means always open, and a NULL end date means it never expires. Below is a list of presents one might wish to add: -| Game Version | Item ID | Item Kind | Item Description | Present Description | -|--------------|---------|-----------|-------------------------------------------------|------------------------------------------------| -| BUDDiES (21) | 409505 | Icon (3) | 旅行スタンプ(月面基地) (Travel Stamp - Moon Base) | Officially obtained on the webui with a serial | -| | | | | number, for project raputa | +| Game Version | Item ID | Item Kind | Item Description | Present Description | +|--------------|---------|----------------------|--------------------------------------------|----------------------------------------------------------------------------| +| BUDDiES (21) | 409505 | Icon (3) | 旅行スタンプ(月面基地) (Travel Stamp - Moon Base) | Officially obtained on the webui with a serial number, for project raputa | +| PRiSM (23) | 3 | KaleidxScopeKey (15) | 紫の鍵 (Purple Key) | Officially obtained on the webui with a serial number, for KaleidxScope | ### Versions From ecd4cc205e7553e73dc20ea4b57bc14b020629dc Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Tue, 8 Apr 2025 08:34:21 +0800 Subject: [PATCH 112/130] Add new KaleidxScope Condition handle method --- titles/mai2/const.py | 11 ++++++++++- titles/mai2/prism.py | 45 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/titles/mai2/const.py b/titles/mai2/const.py index df35da3..6a53324 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -87,7 +87,16 @@ class Mai2Constants: "maimai DX BUDDiES PLUS", "maimai DX PRiSM" ) - + KALEIDXSCOPE_KEY_CONDITION={ + 1: [11009, 11008, 11100, 11097, 11098, 11099, 11163, 11162, 11161, 11228, 11229, 11231, 11463, 11464, 11465, 11538, 11539, 11541, 11620, 11622, 11623, 11737, 11738, 11164, 11230, 11466, 11540, 11621, 11739], + #青の扉: Played 29 songs + 2: [11102, 11234, 11300, 11529, 11542, 11612], + #白の扉: set Frame as "Latent Kingdom" (459504), play 3 or 4 songs by the composer 大国奏音 in 1 pc + 3: [], + #紫の扉: need to enter redeem code 51090942171709440000 + 4: [11023, 11106, 11221, 11222, 11300, 11374, 11458, 11523, 11619, 11663, 11746], + #青の扉: Played 11 songs + } MAI_VERSION_LUT = { "100": VER_MAIMAI, "110": VER_MAIMAI_PLUS, diff --git a/titles/mai2/prism.py b/titles/mai2/prism.py index 5db7c8a..95ebb74 100644 --- a/titles/mai2/prism.py +++ b/titles/mai2/prism.py @@ -43,12 +43,53 @@ class Mai2Prism(Mai2BuddiesPlus): {"gateId": 2, "phaseId": 6}, {"gateId": 3, "phaseId": 6}, {"gateId": 4, "phaseId": 6}, - {"gateId": 5, "phaseId": 6}, - {"gateId": 6, "phaseId": 6} ] } async def handle_get_user_kaleidx_scope_api_request(self, data: Dict) -> Dict: + # kaleidxscope keyget condition judgement + # player may get key before GateFound + for gate in range(1,5): + if gate == 1 or gate == 4: + condition_satisfy = 0 + for condition in Mai2Constants.KALEIDXSCOPE_KEY_CONDITION[gate]: + score_list = await self.data.score.get_best_scores(user_id=data["userId"], song_id=condition) + if score_list: + condition_satisfy = condition_satisfy + 1 + if len(Mai2Constants.KALEIDXSCOPE_KEY_CONDITION[gate]) == condition_satisfy: + new_kaleidxscope = {'gateId': gate, "isKeyFound": True} + await self.data.score.put_user_kaleidxscope(data["userId"], new_kaleidxscope) + + elif gate == 2: + user_profile = await self.data.profile.get_profile_detail(user_id=data["userId"], version=self.version) + user_frame = user_profile["frameId"] + if user_frame == 459504: + playlogs = await self.data.score.get_playlogs(user_id=data["userId"], idx=0, limit=0) + + playlog_dict = {} + for playlog in playlogs: + playlog_id = playlog["playlogId"] + if playlog_id not in playlog_dict: + playlog_dict[playlog_id] = [] + playlog_dict[playlog_id].append(playlog["musicId"]) + valid_playlogs = [] + allowed_music = set(Mai2Constants.KALEIDXSCOPE_KEY_CONDITION[2]) + for playlog_id, music_ids in playlog_dict.items(): + + if len(music_ids) != len(set(music_ids)): + continue + all_valid = True + for mid in music_ids: + if mid not in allowed_music: + all_valid = False + break + if all_valid: + valid_playlogs.append(playlog_id) + + if valid_playlogs: + new_kaleidxscope = {'gateId': 2, "isKeyFound": True} + await self.data.score.put_user_kaleidxscope(data["userId"], new_kaleidxscope) + kaleidxscope = await self.data.score.get_user_kaleidxscope_list(data["userId"]) if kaleidxscope is None: From a7077fb41c8d20c1421694a0985069112085776b Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Mon, 7 Apr 2025 23:16:06 -0400 Subject: [PATCH 113/130] mai2: add prism to version lut --- titles/mai2/const.py | 1 + 1 file changed, 1 insertion(+) diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 6a53324..99642b2 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -124,6 +124,7 @@ class Mai2Constants: "135": VER_MAIMAI_DX_FESTIVAL_PLUS, "140": VER_MAIMAI_DX_BUDDIES, "145": VER_MAIMAI_DX_BUDDIES_PLUS, + "150": VER_MAIMAI_DX_PRISM } @classmethod From e16bfc713aaa5dbfc6a85c0ea5f194f054d97d68 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 8 Apr 2025 00:41:49 -0400 Subject: [PATCH 114/130] chuni: add opt to reader --- .../ae364c078429_chuni_nameplate_add_opt.py | 30 +++++++ titles/chuni/read.py | 21 +++-- titles/chuni/schema/static.py | 84 +++++++++++++------ 3 files changed, 102 insertions(+), 33 deletions(-) create mode 100644 core/data/alembic/versions/ae364c078429_chuni_nameplate_add_opt.py diff --git a/core/data/alembic/versions/ae364c078429_chuni_nameplate_add_opt.py b/core/data/alembic/versions/ae364c078429_chuni_nameplate_add_opt.py new file mode 100644 index 0000000..b6f61bd --- /dev/null +++ b/core/data/alembic/versions/ae364c078429_chuni_nameplate_add_opt.py @@ -0,0 +1,30 @@ +"""chuni_nameplate_add_opt + +Revision ID: ae364c078429 +Revises: 5cf98cfe52ad +Create Date: 2025-04-08 00:22:22.370660 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'ae364c078429' +down_revision = '5cf98cfe52ad' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('chuni_static_nameplate', sa.Column('opt', sa.BIGINT(), nullable=True)) + op.create_foreign_key(None, 'chuni_static_nameplate', 'chuni_static_opt', ['opt'], ['id'], onupdate='cascade', ondelete='SET NULL') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("chuni_static_nameplate_ibfk_1", 'chuni_static_nameplate', type_='foreignkey') + op.drop_column('chuni_static_nameplate', 'opt') + # ### end Alembic commands ### diff --git a/titles/chuni/read.py b/titles/chuni/read.py index fb81d1d..bd6ff07 100644 --- a/titles/chuni/read.py +++ b/titles/chuni/read.py @@ -78,7 +78,7 @@ class ChuniReader(BaseReader): is_enabled = True if (disableFlag is None or disableFlag.text == "false") else False result = await self.data.static.put_login_bonus_preset( - self.version, id, name, is_enabled + self.version, id, name, is_enabled, opt_id ) if result is not None: @@ -125,6 +125,7 @@ class ChuniReader(BaseReader): item_num, need_login_day_count, login_bonus_category_type, + opt_id ) if result is not None: @@ -149,7 +150,7 @@ class ChuniReader(BaseReader): event_type = substances.find("type").text result = await self.data.static.put_event( - self.version, id, event_type, name + self.version, id, event_type, name, opt_id ) if result is not None: self.logger.info(f"Inserted event {id}") @@ -221,6 +222,7 @@ class ChuniReader(BaseReader): genre, jacket_path, we_chara, + opt_id ) if result is not None: @@ -254,6 +256,7 @@ class ChuniReader(BaseReader): expirationDays, consumeType, sellingAppeal, + opt_id ) if result is not None: @@ -286,7 +289,7 @@ class ChuniReader(BaseReader): self.copy_image(texturePath, f"{root}/{dir}", "titles/chuni/img/avatar/") result = await self.data.static.put_avatar( - self.version, id, name, category, iconPath, texturePath, is_enabled, defaultHave, sortName + self.version, id, name, category, iconPath, texturePath, is_enabled, defaultHave, sortName, opt_id ) if result is not None: @@ -315,7 +318,7 @@ class ChuniReader(BaseReader): self.copy_image(texturePath, f"{root}/{dir}", "titles/chuni/img/nameplate/") result = await self.data.static.put_nameplate( - self.version, id, name, texturePath, is_enabled, defaultHave, sortName + self.version, id, name, texturePath, is_enabled, defaultHave, sortName, opt_id ) if result is not None: @@ -340,7 +343,7 @@ class ChuniReader(BaseReader): defaultHave = xml_root.find("defaultHave").text == 'true' result = await self.data.static.put_trophy( - self.version, id, name, rareType, is_enabled, defaultHave + self.version, id, name, rareType, is_enabled, defaultHave, opt_id ) if result is not None: @@ -387,7 +390,7 @@ class ChuniReader(BaseReader): self.logger.warning(f"Unable to location character {id} images") result = await self.data.static.put_character( - self.version, id, name, sortName, worksName, rareType, imagePath1, imagePath2, imagePath3, is_enabled, defaultHave + self.version, id, name, sortName, worksName, rareType, imagePath1, imagePath2, imagePath3, is_enabled, defaultHave, opt_id ) if result is not None: @@ -415,7 +418,7 @@ class ChuniReader(BaseReader): is_enabled = True if (disableFlag is None or disableFlag.text == "false") else False result = await self.data.static.put_map_icon( - self.version, id, name, sortName, iconPath, is_enabled, defaultHave + self.version, id, name, sortName, iconPath, is_enabled, defaultHave, opt_id ) if result is not None: @@ -443,7 +446,7 @@ class ChuniReader(BaseReader): is_enabled = True if (disableFlag is None or disableFlag.text == "false") else False result = await self.data.static.put_system_voice( - self.version, id, name, sortName, imagePath, is_enabled, defaultHave + self.version, id, name, sortName, imagePath, is_enabled, defaultHave, opt_id ) if result is not None: @@ -490,6 +493,8 @@ class ChuniReader(BaseReader): if not opt_id: self.logger.error(f"Failed to put opt folder info for {opt_folder}") return None + else: + opt_id = opt_id['id'] self.logger.info(f"Opt folder {opt_folder} (Database ID {opt_id}) contains {data_config['Version']['Name']} v{data_config['Version']['VerMajor']}.{data_config['Version']['VerMinor']}.{opt_seq}") return opt_id diff --git a/titles/chuni/schema/static.py b/titles/chuni/schema/static.py index 0f7dc4a..f4f0f9f 100644 --- a/titles/chuni/schema/static.py +++ b/titles/chuni/schema/static.py @@ -13,6 +13,7 @@ from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.dialects.mysql import insert from datetime import datetime +from sqlalchemy.sql.functions import coalesce from core.data.schema import BaseData, metadata @@ -107,6 +108,7 @@ nameplate = Table( Column("isEnabled", Boolean, server_default="1"), Column("defaultHave", Boolean, server_default="0"), Column("sortName", String(255)), + Column("opt", ForeignKey("chuni_static_opt.id", ondelete="SET NULL", onupdate="cascade")), UniqueConstraint("version", "nameplateId", name="chuni_static_nameplate_uk"), mysql_charset="utf8mb4", ) @@ -299,6 +301,7 @@ class ChuniStaticData(BaseData): item_num: int, need_login_day_count: int, login_bonus_category_type: int, + opt_id: int = None ) -> Optional[int]: sql = insert(login_bonus).values( version=version, @@ -310,6 +313,7 @@ class ChuniStaticData(BaseData): itemNum=item_num, needLoginDayCount=need_login_day_count, loginBonusCategoryType=login_bonus_category_type, + opt=coalesce(login_bonus.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -318,6 +322,7 @@ class ChuniStaticData(BaseData): itemNum=item_num, needLoginDayCount=need_login_day_count, loginBonusCategoryType=login_bonus_category_type, + opt=coalesce(login_bonus.c.opt, opt_id) ) result = await self.execute(conflict) @@ -359,17 +364,19 @@ class ChuniStaticData(BaseData): return result.fetchone() async def put_login_bonus_preset( - self, version: int, preset_id: int, preset_name: str, isEnabled: bool + self, version: int, preset_id: int, preset_name: str, isEnabled: bool, opt_id: int = None ) -> Optional[int]: sql = insert(login_bonus_preset).values( presetId=preset_id, version=version, presetName=preset_name, isEnabled=isEnabled, + opt=coalesce(login_bonus_preset.c.opt, opt_id) ) - - conflict = sql.on_duplicate_key_update( - presetName=preset_name, isEnabled=isEnabled + + # Chuni has a habbit of including duplicates in it's opt files, so only update opt if it's null + conflict = sql.on_duplicate_key_update( + presetName=preset_name, isEnabled=isEnabled, opt=coalesce(login_bonus_preset.c.opt, opt_id) ) result = await self.execute(conflict) @@ -393,13 +400,13 @@ class ChuniStaticData(BaseData): return result.fetchall() async def put_event( - self, version: int, event_id: int, type: int, name: str + self, version: int, event_id: int, type: int, name: str, opt_id: int = None ) -> Optional[int]: sql = insert(events).values( - version=version, eventId=event_id, type=type, name=name + version=version, eventId=event_id, type=type, name=name, opt=coalesce(events.c.opt, opt_id) ) - conflict = sql.on_duplicate_key_update(name=name) + conflict = sql.on_duplicate_key_update(name=name, opt=coalesce(events.c.opt, opt_id)) result = await self.execute(conflict) if result is None: @@ -467,6 +474,7 @@ class ChuniStaticData(BaseData): genre: str, jacketPath: str, we_tag: str, + opt_id: int = None ) -> Optional[int]: sql = insert(music).values( version=version, @@ -478,6 +486,7 @@ class ChuniStaticData(BaseData): genre=genre, jacketPath=jacketPath, worldsEndTag=we_tag, + opt=coalesce(music.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -487,6 +496,7 @@ class ChuniStaticData(BaseData): genre=genre, jacketPath=jacketPath, worldsEndTag=we_tag, + opt=coalesce(music.c.opt, opt_id) ) result = await self.execute(conflict) @@ -502,6 +512,7 @@ class ChuniStaticData(BaseData): expiration_days: int, consume_type: int, selling_appeal: bool, + opt_id: int = None ) -> Optional[int]: sql = insert(charge).values( version=version, @@ -510,6 +521,7 @@ class ChuniStaticData(BaseData): expirationDays=expiration_days, consumeType=consume_type, sellingAppeal=selling_appeal, + opt=coalesce(charge.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -517,6 +529,7 @@ class ChuniStaticData(BaseData): expirationDays=expiration_days, consumeType=consume_type, sellingAppeal=selling_appeal, + opt=coalesce(charge.c.opt, opt_id) ) result = await self.execute(conflict) @@ -584,7 +597,8 @@ class ChuniStaticData(BaseData): texturePath: str, isEnabled: int, defaultHave: int, - sortName: str + sortName: str, + opt_id: int = None ) -> Optional[int]: sql = insert(avatar).values( version=version, @@ -595,7 +609,8 @@ class ChuniStaticData(BaseData): texturePath=texturePath, isEnabled=isEnabled, defaultHave=defaultHave, - sortName=sortName + sortName=sortName, + opt=coalesce(avatar.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -605,7 +620,8 @@ class ChuniStaticData(BaseData): texturePath=texturePath, isEnabled=isEnabled, defaultHave=defaultHave, - sortName=sortName + sortName=sortName, + opt=coalesce(avatar.c.opt, opt_id) ) result = await self.execute(conflict) @@ -632,7 +648,8 @@ class ChuniStaticData(BaseData): texturePath: str, isEnabled: int, defaultHave: int, - sortName: str + sortName: str, + opt_id: int = None ) -> Optional[int]: sql = insert(nameplate).values( version=version, @@ -641,7 +658,8 @@ class ChuniStaticData(BaseData): texturePath=texturePath, isEnabled=isEnabled, defaultHave=defaultHave, - sortName=sortName + sortName=sortName, + opt=coalesce(nameplate.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -649,7 +667,8 @@ class ChuniStaticData(BaseData): texturePath=texturePath, isEnabled=isEnabled, defaultHave=defaultHave, - sortName=sortName + sortName=sortName, + opt=coalesce(nameplate.c.opt, opt_id) ) result = await self.execute(conflict) @@ -676,6 +695,7 @@ class ChuniStaticData(BaseData): rareType: int, isEnabled: int, defaultHave: int, + opt_id: int = None ) -> Optional[int]: sql = insert(trophy).values( version=version, @@ -683,14 +703,16 @@ class ChuniStaticData(BaseData): name=name, rareType=rareType, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(trophy.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( name=name, rareType=rareType, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(trophy.c.opt, opt_id) ) result = await self.execute(conflict) @@ -718,6 +740,7 @@ class ChuniStaticData(BaseData): iconPath: str, isEnabled: int, defaultHave: int, + opt_id: int = None ) -> Optional[int]: sql = insert(map_icon).values( version=version, @@ -726,7 +749,8 @@ class ChuniStaticData(BaseData): sortName=sortName, iconPath=iconPath, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(map_icon.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -734,7 +758,8 @@ class ChuniStaticData(BaseData): sortName=sortName, iconPath=iconPath, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(map_icon.c.opt, opt_id) ) result = await self.execute(conflict) @@ -762,6 +787,7 @@ class ChuniStaticData(BaseData): imagePath: str, isEnabled: int, defaultHave: int, + opt_id: int = None ) -> Optional[int]: sql = insert(system_voice).values( version=version, @@ -770,7 +796,8 @@ class ChuniStaticData(BaseData): sortName=sortName, imagePath=imagePath, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(system_voice.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -778,7 +805,8 @@ class ChuniStaticData(BaseData): sortName=sortName, imagePath=imagePath, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(system_voice.c.opt, opt_id) ) result = await self.execute(conflict) @@ -809,7 +837,8 @@ class ChuniStaticData(BaseData): imagePath2: str, imagePath3: str, isEnabled: int, - defaultHave: int + defaultHave: int, + opt_id: int = None ) -> Optional[int]: sql = insert(character).values( version=version, @@ -822,7 +851,8 @@ class ChuniStaticData(BaseData): imagePath2=imagePath2, imagePath3=imagePath3, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(character.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -834,7 +864,8 @@ class ChuniStaticData(BaseData): imagePath2=imagePath2, imagePath3=imagePath3, isEnabled=isEnabled, - defaultHave=defaultHave + defaultHave=defaultHave, + opt=coalesce(character.c.opt, opt_id) ) result = await self.execute(conflict) @@ -858,12 +889,14 @@ class ChuniStaticData(BaseData): version: int, gacha_id: int, gacha_name: int, + opt_id: int = None, **gacha_data, ) -> Optional[int]: sql = insert(gachas).values( version=version, gachaId=gacha_id, gachaName=gacha_name, + opt=coalesce(gachas.c.opt, opt_id), **gacha_data, ) @@ -871,6 +904,7 @@ class ChuniStaticData(BaseData): version=version, gachaId=gacha_id, gachaName=gacha_name, + opt=coalesce(gachas.c.opt, opt_id), **gacha_data, ) @@ -940,10 +974,10 @@ class ChuniStaticData(BaseData): return None return result.fetchone() - async def put_card(self, version: int, card_id: int, **card_data) -> Optional[int]: - sql = insert(cards).values(version=version, cardId=card_id, **card_data) + async def put_card(self, version: int, card_id: int, opt_id: int = None,**card_data) -> Optional[int]: + sql = insert(cards).values(version=version, cardId=card_id, opt=coalesce(cards.c.opt, opt_id), **card_data) - conflict = sql.on_duplicate_key_update(**card_data) + conflict = sql.on_duplicate_key_update(opt=coalesce(cards.c.opt, opt_id), **card_data) result = await self.execute(conflict) if result is None: From 47affd898f000fde83e8305bb6acfc8d704d037f Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 8 Apr 2025 17:42:17 -0400 Subject: [PATCH 115/130] mai2: add opts to reader --- titles/mai2/read.py | 72 +++++++++++++++++++----- titles/mai2/schema/static.py | 103 ++++++++++++++++++++++++++++++++--- 2 files changed, 154 insertions(+), 21 deletions(-) diff --git a/titles/mai2/read.py b/titles/mai2/read.py index d9450ac..a84e7be 100644 --- a/titles/mai2/read.py +++ b/titles/mai2/read.py @@ -1,20 +1,16 @@ -from decimal import Decimal -import logging import os import re import xml.etree.ElementTree as ET -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional from Crypto.Cipher import AES import zlib import codecs from core.config import CoreConfig -from core.data import Data from read import BaseReader from titles.mai2.const import Mai2Constants from titles.mai2.database import Mai2Data - class Mai2Reader(BaseReader): def __init__( self, @@ -46,10 +42,11 @@ class Mai2Reader(BaseReader): for dir in data_dirs: self.logger.info(f"Read from {dir}") - await self.get_events(f"{dir}/event") + this_opt_id = await self.read_opt_info(dir) + await self.get_events(f"{dir}/event", this_opt_id) await self.disable_events(f"{dir}/information", f"{dir}/scoreRanking") - await self.read_music(f"{dir}/music") - await self.read_tickets(f"{dir}/ticket") + await self.read_music(f"{dir}/music", this_opt_id) + await self.read_tickets(f"{dir}/ticket", this_opt_id) else: if not os.path.exists(f"{self.bin_dir}/tables"): @@ -179,7 +176,7 @@ class Mai2Reader(BaseReader): self.logger.warning("Failed load table content, skipping") return - async def get_events(self, base_dir: str) -> None: + async def get_events(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading events from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -193,7 +190,7 @@ class Mai2Reader(BaseReader): event_type = int(troot.find("infoType").text) await self.data.static.put_game_event( - self.version, event_type, id, name + self.version, event_type, id, name, opt_id ) self.logger.info(f"Added event {id}...") @@ -255,7 +252,7 @@ class Mai2Reader(BaseReader): await self.data.static.toggle_game_event(self.version, event_id, toggle=False) self.logger.info(f"Disabled event {event_id}...") - async def read_music(self, base_dir: str) -> None: + async def read_music(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading music from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -296,13 +293,14 @@ class Mai2Reader(BaseReader): added_ver, diff_num, note_designer, + opt_id ) self.logger.info( f"Added music id {song_id} chart {chart_id}" ) - async def read_tickets(self, base_dir: str) -> None: + async def read_tickets(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading tickets from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -317,7 +315,7 @@ class Mai2Reader(BaseReader): price = int(troot.find("creditNum").text) await self.data.static.put_game_ticket( - self.version, id, ticket_type, price, name + self.version, id, ticket_type, price, name, opt_id ) self.logger.info(f"Added ticket {id}...") @@ -341,3 +339,51 @@ class Mai2Reader(BaseReader): if scores is None or text is None: return # TODO + + async def read_opt_info(self, directory: str) -> Optional[int]: + datacfg_file = os.path.join(directory, "DataConfig.xml") + if not os.path.exists(datacfg_file): + self.logger.warning(f"{datacfg_file} does not contain DataConfig.xml, opt info will not be read") + return None + + with open(datacfg_file, encoding="utf-8") as f: + troot = ET.fromstring(f.read()) + + if troot.find("DataConfig/version") is None: + self.logger.warning(f"{directory}/DataConfig.xml contains no Version section, opt info will not be read") + return None + + ver_maj = troot.find("DataConfig/version/major") + ver_min = troot.find("DataConfig/version/minor") + ver_rel = troot.find("DataConfig/version/release") + cm_maj = troot.find("DataConfig/cardMakerVersion/major") + cm_min = troot.find("DataConfig/cardMakerVersion/minor") + cm_rel = troot.find("DataConfig/cardMakerVersion/release") + + if ver_maj is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no major item in the Version section, opt info will not be read") + return None + + if ver_min is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no minor item in the Version section, opt info will not be read") + return None + + if ver_rel is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no release item in the Version section, opt info will not be read") + return None + + opt_folder = os.path.basename(os.path.normpath(directory)) + opt_id = await self.data.static.get_opt_by_version_folder(self.version, opt_folder) + + if not opt_id: + opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel else None) + if not opt_id: + self.logger.error(f"Failed to put opt folder info for {opt_folder}") + return None + else: + opt_id = opt_id['id'] + + self.logger.info( + f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj else 'None'}.{cm_min.text if cm_min else 'None'}.{cm_rel.text if cm_rel else 'None'})" + ) + return opt_id diff --git a/titles/mai2/schema/static.py b/titles/mai2/schema/static.py index 33b93c6..29e020e 100644 --- a/titles/mai2/schema/static.py +++ b/titles/mai2/schema/static.py @@ -7,6 +7,7 @@ from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row from sqlalchemy.dialects.mysql import insert +from sqlalchemy.sql.functions import coalesce from datetime import datetime opts = Table( @@ -92,16 +93,17 @@ cards = Table( class Mai2StaticData(BaseData): async def put_game_event( - self, version: int, type: int, event_id: int, name: str + self, version: int, type: int, event_id: int, name: str, opt_id: int = None ) -> Optional[int]: sql = insert(event).values( version=version, type=type, eventId=event_id, name=name, + opt=coalesce(event.c.opt, opt_id) ) - conflict = sql.on_duplicate_key_update(eventId=event_id) + conflict = sql.on_duplicate_key_update(eventId=event_id, opt=coalesce(event.c.opt, opt_id)) result = await self.execute(conflict) if result is None: @@ -154,6 +156,7 @@ class Mai2StaticData(BaseData): added_version: str, difficulty: float, note_designer: str, + opt_id: int = None ) -> None: sql = insert(music).values( version=version, @@ -166,6 +169,7 @@ class Mai2StaticData(BaseData): addedVersion=added_version, difficulty=difficulty, noteDesigner=note_designer, + opt=coalesce(music.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -176,6 +180,7 @@ class Mai2StaticData(BaseData): addedVersion=added_version, difficulty=difficulty, noteDesigner=note_designer, + opt=coalesce(music.c.opt, opt_id) ) result = await self.execute(conflict) @@ -191,6 +196,7 @@ class Mai2StaticData(BaseData): ticket_type: int, ticket_price: int, name: str, + opt_id: int = None ) -> Optional[int]: sql = insert(ticket).values( version=version, @@ -198,11 +204,10 @@ class Mai2StaticData(BaseData): kind=ticket_type, price=ticket_price, name=name, + opt=coalesce(ticket.c.opt, opt_id) ) - conflict = sql.on_duplicate_key_update(price=ticket_price) - - conflict = sql.on_duplicate_key_update(price=ticket_price) + conflict = sql.on_duplicate_key_update(price=ticket_price, opt=coalesce(ticket.c.opt, opt_id)) result = await self.execute(conflict) if result is None: @@ -247,12 +252,12 @@ class Mai2StaticData(BaseData): return None return result.fetchone() - async def put_card(self, version: int, card_id: int, card_name: str, **card_data) -> int: + async def put_card(self, version: int, card_id: int, card_name: str, opt_id: int = None, **card_data) -> int: sql = insert(cards).values( - version=version, cardId=card_id, cardName=card_name, **card_data + version=version, cardId=card_id, cardName=card_name, opt=coalesce(cards.c.opt, opt_id) **card_data ) - conflict = sql.on_duplicate_key_update(**card_data) + conflict = sql.on_duplicate_key_update(opt=coalesce(cards.c.opt, opt_id), **card_data) result = await self.execute(conflict) if result is None: @@ -282,3 +287,85 @@ class Mai2StaticData(BaseData): result = await self.execute(event.update(event.c.id == table_id).values(enabled=is_enable, startDate = start_date)) if not result: self.logger.error(f"Failed to update event {table_id} - {is_enable} {start_date}") + + async def put_opt(self, version: int, folder: str, sequence: int, cm_seq: int = None) -> Optional[int]: + sql = insert(opts).values(version=version, name=folder, sequence=sequence, cmReleaseVer=cm_seq) + + conflict = sql.on_duplicate_key_update(sequence=sequence, whenRead=datetime.now()) + + result = await self.execute(conflict) + if result is None: + self.logger.warning(f"Failed to insert opt! version {version} folder {folder} sequence {sequence}") + return None + return result.lastrowid + + async def get_opt_by_version_folder(self, version: int, folder: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.name == folder, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opt_by_version_sequence(self, version: int, sequence: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.sequence == sequence, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opts_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(opts.c.version == version)) + + if result is None: + return None + return result.fetchall() + + async def get_opts_enabled_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + ))) + + if result is None: + return None + return result.fetchall() + + async def get_latest_enabled_opt_by_version(self, version: int) -> Optional[Row]: + result = await self.execute( + opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + )).order_by(opts.c.sequence.desc()) + ) + + if result is None: + return None + return result.fetchone() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def set_opt_enabled(self, opt_id: int, enabled: bool) -> bool: + result = await self.execute(opts.update(opts.c.id == opt_id).values(isEnable=enabled)) + + if result is None: + self.logger.error(f"Failed to set opt enabled status to {enabled} for opt {opt_id}") + return False + return True From 9a14e543283750fe9c4d6b28d0077f5cf012a2ac Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 8 Apr 2025 17:59:19 -0400 Subject: [PATCH 116/130] ongeki: add opts to reader --- titles/ongeki/read.py | 73 +++++++++++++++++++++++++++++----- titles/ongeki/schema/static.py | 37 ++++++++++------- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index ed3043f..594689e 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -58,12 +58,13 @@ class OngekiReader(BaseReader): data_dirs += self.get_data_directories(self.opt_dir) for dir in data_dirs: - await self.read_events(f"{dir}/event") - await self.read_music(f"{dir}/music") - await self.read_card(f"{dir}/card") - await self.read_reward(f"{dir}/reward") + this_opt_id = await self.read_opt_info(dir) + await self.read_events(f"{dir}/event", this_opt_id) + await self.read_music(f"{dir}/music", this_opt_id) + await self.read_card(f"{dir}/card", this_opt_id) + await self.read_reward(f"{dir}/reward", this_opt_id) - async def read_card(self, base_dir: str) -> None: + async def read_card(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading cards from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -75,6 +76,7 @@ class OngekiReader(BaseReader): card_id = int(troot.find("Name").find("id").text) # skip already existing cards + # Hay1tsme 2025/04/08: What is this for, and why does it only check for BM cards? if ( await self.data.static.get_card( OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, card_id @@ -108,6 +110,7 @@ class OngekiReader(BaseReader): await self.data.static.put_card( self.parse_version(troot), card_id, + opt_id, name=name, charaId=chara_id, nickName=nick_name, @@ -122,7 +125,7 @@ class OngekiReader(BaseReader): ) self.logger.info(f"Added card {card_id}") - async def read_events(self, base_dir: str) -> None: + async def read_events(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading events from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -140,10 +143,10 @@ class OngekiReader(BaseReader): if troot.find("EventType").text == "MissionEvent": name = (troot.find("Event").find("MissionName").find("str").text) - await self.data.static.put_event(self.version, id, event_type, name) + await self.data.static.put_event(self.version, id, event_type, name, opt_id) self.logger.info(f"Added event {id}") - async def read_music(self, base_dir: str) -> None: + async def read_music(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading music from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -178,11 +181,11 @@ class OngekiReader(BaseReader): ) await self.data.static.put_chart( - version, song_id, chart_id, title, artist, genre, level + version, song_id, chart_id, title, artist, genre, level, opt_id ) self.logger.info(f"Added song {song_id} chart {chart_id}") - async def read_reward(self, base_dir: str) -> None: + async def read_reward(self, base_dir: str, opt_id: int = None) -> None: self.logger.info(f"Reading rewards from {base_dir}...") for root, dirs, files in os.walk(base_dir): @@ -204,5 +207,53 @@ class OngekiReader(BaseReader): itemKind = OngekiConstants.REWARD_TYPES[troot.find("ItemType").text].value itemId = troot.find("RewardItem").find("ItemName").find("id").text - await self.data.static.put_reward(self.version, rewardId, rewardname, itemKind, itemId) + await self.data.static.put_reward(self.version, rewardId, rewardname, itemKind, itemId, opt_id) self.logger.info(f"Added reward {rewardId}") + + async def read_opt_info(self, directory: str) -> Optional[int]: + datacfg_file = os.path.join(directory, "DataConfig.xml") + if not os.path.exists(datacfg_file): + self.logger.warning(f"{datacfg_file} does not contain DataConfig.xml, opt info will not be read") + return None + + with open(datacfg_file, encoding="utf-8") as f: + troot = ET.fromstring(f.read()) + + if troot.find("DataConfig/version") is None: + self.logger.warning(f"{directory}/DataConfig.xml contains no Version section, opt info will not be read") + return None + + ver_maj = troot.find("DataConfig/version/major") + ver_min = troot.find("DataConfig/version/minor") + ver_rel = troot.find("DataConfig/version/release") + cm_maj = troot.find("DataConfig/cardMakerVersion/major") + cm_min = troot.find("DataConfig/cardMakerVersion/minor") + cm_rel = troot.find("DataConfig/cardMakerVersion/release") + + if ver_maj is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no major item in the Version section, opt info will not be read") + return None + + if ver_min is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no minor item in the Version section, opt info will not be read") + return None + + if ver_rel is None: # Probably not worth checking that the other sections exist + self.logger.warning(f"{datacfg_file} contains no release item in the Version section, opt info will not be read") + return None + + opt_folder = os.path.basename(os.path.normpath(directory)) + opt_id = await self.data.static.get_opt_by_version_folder(self.version, opt_folder) + + if not opt_id: + opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel else None) + if not opt_id: + self.logger.error(f"Failed to put opt folder info for {opt_folder}") + return None + else: + opt_id = opt_id['id'] + + self.logger.info( + f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj else 'None'}.{cm_min.text if cm_min else 'None'}.{cm_rel.text if cm_rel else 'None'})" + ) + return opt_id diff --git a/titles/ongeki/schema/static.py b/titles/ongeki/schema/static.py index 8609f5c..a784f67 100644 --- a/titles/ongeki/schema/static.py +++ b/titles/ongeki/schema/static.py @@ -5,6 +5,7 @@ from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row from sqlalchemy.dialects.mysql import insert +from sqlalchemy.sql.functions import coalesce from core.data.schema import BaseData, metadata from core.data.schema.arcade import machine @@ -212,10 +213,10 @@ game_point = Table( ) class OngekiStaticData(BaseData): - async def put_card(self, version: int, card_id: int, **card_data) -> Optional[int]: - sql = insert(cards).values(version=version, cardId=card_id, **card_data) + async def put_card(self, version: int, card_id: int, opt_id: int = None, **card_data) -> Optional[int]: + sql = insert(cards).values(version=version, cardId=card_id, opt=coalesce(cards.c.opt, opt_id), **card_data) - conflict = sql.on_duplicate_key_update(**card_data) + conflict = sql.on_duplicate_key_update(opt=coalesce(cards.c.opt, opt_id), **card_data) result = await self.execute(conflict) if result is None: @@ -342,7 +343,7 @@ class OngekiStaticData(BaseData): return result.fetchall() async def put_event( - self, version: int, event_id: int, event_type: int, event_name: str + self, version: int, event_id: int, event_type: int, event_name: str, opt_id: int = None ) -> Optional[int]: sql = insert(events).values( version=version, @@ -350,10 +351,11 @@ class OngekiStaticData(BaseData): type=event_type, name=event_name, endDate=f"2038-01-01 00:00:00", + opt=coalesce(events.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( - name=event_name, + name=event_name, opt=coalesce(events.c.opt, opt_id) ) result = await self.execute(conflict) @@ -399,6 +401,7 @@ class OngekiStaticData(BaseData): artist: str, genre: str, level: float, + opt_id: int = None ) -> Optional[int]: sql = insert(music).values( version=version, @@ -408,6 +411,7 @@ class OngekiStaticData(BaseData): artist=artist, genre=genre, level=level, + opt=coalesce(music.c.opt, opt_id) ) conflict = sql.on_duplicate_key_update( @@ -415,6 +419,7 @@ class OngekiStaticData(BaseData): artist=artist, genre=genre, level=level, + opt=coalesce(music.c.opt, opt_id) ) result = await self.execute(conflict) @@ -449,17 +454,21 @@ class OngekiStaticData(BaseData): return None return result.fetchone() - async def put_reward(self, version: int, rewardId: int, rewardname: str, itemKind: int, itemId: int) -> Optional[int]: + async def put_reward(self, version: int, rewardId: int, rewardname: str, itemKind: int, itemId: int, opt_id: int = None) -> Optional[int]: sql = insert(rewards).values( - version=version, - rewardId=rewardId, - rewardname=rewardname, - itemKind=itemKind, - itemId=itemId, - ) + version=version, + rewardId=rewardId, + rewardname=rewardname, + itemKind=itemKind, + itemId=itemId, + opt=coalesce(rewards.c.opt, opt_id) + ) + conflict = sql.on_duplicate_key_update( - rewardname=rewardname, - ) + rewardname=rewardname, + opt=coalesce(rewards.c.opt, opt_id) + ) + result = await self.execute(conflict) if result is None: self.logger.warning(f"Failed to insert reward! reward_id: {rewardId}") From c955c1ae37445ce9a261b6bcb6a3081a5a13b6b2 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Tue, 8 Apr 2025 23:45:15 -0400 Subject: [PATCH 117/130] mai2: fix opt reader --- titles/mai2/read.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/titles/mai2/read.py b/titles/mai2/read.py index a84e7be..1c86518 100644 --- a/titles/mai2/read.py +++ b/titles/mai2/read.py @@ -349,16 +349,16 @@ class Mai2Reader(BaseReader): with open(datacfg_file, encoding="utf-8") as f: troot = ET.fromstring(f.read()) - if troot.find("DataConfig/version") is None: + if troot.find("version") is None: self.logger.warning(f"{directory}/DataConfig.xml contains no Version section, opt info will not be read") return None - ver_maj = troot.find("DataConfig/version/major") - ver_min = troot.find("DataConfig/version/minor") - ver_rel = troot.find("DataConfig/version/release") - cm_maj = troot.find("DataConfig/cardMakerVersion/major") - cm_min = troot.find("DataConfig/cardMakerVersion/minor") - cm_rel = troot.find("DataConfig/cardMakerVersion/release") + ver_maj = troot.find("version/major") + ver_min = troot.find("version/minor") + ver_rel = troot.find("version/release") + cm_maj = troot.find("cardMakerVersion/major") + cm_min = troot.find("cardMakerVersion/minor") + cm_rel = troot.find("cardMakerVersion/release") if ver_maj is None: # Probably not worth checking that the other sections exist self.logger.warning(f"{datacfg_file} contains no major item in the Version section, opt info will not be read") @@ -376,7 +376,7 @@ class Mai2Reader(BaseReader): opt_id = await self.data.static.get_opt_by_version_folder(self.version, opt_folder) if not opt_id: - opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel else None) + opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel is not None else None) if not opt_id: self.logger.error(f"Failed to put opt folder info for {opt_folder}") return None @@ -384,6 +384,6 @@ class Mai2Reader(BaseReader): opt_id = opt_id['id'] self.logger.info( - f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj else 'None'}.{cm_min.text if cm_min else 'None'}.{cm_rel.text if cm_rel else 'None'})" + f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj is not None else 'None'}.{cm_min.text if cm_min is not None else 'None'}.{cm_rel.text if cm_rel is not None else 'None'})" ) return opt_id From 2640f23a00db696724438c335801799e98ccb4de Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Wed, 9 Apr 2025 00:10:54 -0400 Subject: [PATCH 118/130] ongeki: fix opt reader --- titles/ongeki/read.py | 18 ++--- titles/ongeki/schema/static.py | 120 ++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index 594689e..4797dcb 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -219,16 +219,16 @@ class OngekiReader(BaseReader): with open(datacfg_file, encoding="utf-8") as f: troot = ET.fromstring(f.read()) - if troot.find("DataConfig/version") is None: + if troot.find("version") is None: self.logger.warning(f"{directory}/DataConfig.xml contains no Version section, opt info will not be read") return None - ver_maj = troot.find("DataConfig/version/major") - ver_min = troot.find("DataConfig/version/minor") - ver_rel = troot.find("DataConfig/version/release") - cm_maj = troot.find("DataConfig/cardMakerVersion/major") - cm_min = troot.find("DataConfig/cardMakerVersion/minor") - cm_rel = troot.find("DataConfig/cardMakerVersion/release") + ver_maj = troot.find("version/major") + ver_min = troot.find("version/minor") + ver_rel = troot.find("version/release") + cm_maj = troot.find("cardMakerVersion/major") + cm_min = troot.find("cardMakerVersion/minor") + cm_rel = troot.find("cardMakerVersion/release") if ver_maj is None: # Probably not worth checking that the other sections exist self.logger.warning(f"{datacfg_file} contains no major item in the Version section, opt info will not be read") @@ -246,7 +246,7 @@ class OngekiReader(BaseReader): opt_id = await self.data.static.get_opt_by_version_folder(self.version, opt_folder) if not opt_id: - opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel else None) + opt_id = await self.data.static.put_opt(self.version, opt_folder, int(ver_rel.text), int(cm_rel.text) if cm_rel is not None else None) if not opt_id: self.logger.error(f"Failed to put opt folder info for {opt_folder}") return None @@ -254,6 +254,6 @@ class OngekiReader(BaseReader): opt_id = opt_id['id'] self.logger.info( - f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj else 'None'}.{cm_min.text if cm_min else 'None'}.{cm_rel.text if cm_rel else 'None'})" + f"Opt folder {opt_folder} (Database ID {opt_id}) contains v{ver_maj.text}.{ver_min.text}.{ver_rel.text} (cm v{cm_maj.text if cm_maj is not None else 'None'}.{cm_min.text if cm_min is not None else 'None'}.{cm_rel.text if cm_rel is not None else 'None'})" ) return opt_id diff --git a/titles/ongeki/schema/static.py b/titles/ongeki/schema/static.py index a784f67..bf4af07 100644 --- a/titles/ongeki/schema/static.py +++ b/titles/ongeki/schema/static.py @@ -6,6 +6,7 @@ from sqlalchemy.sql import func, select from sqlalchemy.engine import Row from sqlalchemy.dialects.mysql import insert from sqlalchemy.sql.functions import coalesce +from datetime import datetime from core.data.schema import BaseData, metadata from core.data.schema.arcade import machine @@ -57,7 +58,6 @@ events = Table( mysql_charset="utf8mb4", ) - music = Table( "ongeki_static_music", metadata, @@ -536,3 +536,121 @@ class OngekiStaticData(BaseData): if result is None: return None return result.fetchall() + + async def put_opt(self, version: int, folder: str, sequence: int, cm_seq: int = None) -> Optional[int]: + sql = insert(opts).values(version=version, name=folder, sequence=sequence, cmReleaseVer=cm_seq) + + conflict = sql.on_duplicate_key_update(sequence=sequence, whenRead=datetime.now()) + + result = await self.execute(conflict) + if result is None: + self.logger.warning(f"Failed to insert opt! version {version} folder {folder} sequence {sequence}") + return None + return result.lastrowid + + async def get_opt_by_version_folder(self, version: int, folder: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.name == folder, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opt_by_version_sequence(self, version: int, sequence: str) -> Optional[Row]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.sequence == sequence, + ))) + + if result is None: + return None + return result.fetchone() + + async def get_opts_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(opts.c.version == version)) + + if result is None: + return None + return result.fetchall() + + async def get_opts_enabled_by_version(self, version: int) -> Optional[List[Row]]: + result = await self.execute(opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + ))) + + if result is None: + return None + return result.fetchall() + + async def get_latest_enabled_opt_by_version(self, version: int) -> Optional[Row]: + result = await self.execute( + opts.select(and_( + opts.c.version == version, + opts.c.isEnable == True, + )).order_by(opts.c.sequence.desc()) + ) + + if result is None: + return None + return result.fetchone() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def get_opts(self) -> Optional[List[Row]]: + result = await self.execute(opts.select()) + + if result is None: + return None + return result.fetchall() + + async def set_opt_enabled(self, opt_id: int, enabled: bool) -> bool: + result = await self.execute(opts.update(opts.c.id == opt_id).values(isEnable=enabled)) + + if result is None: + self.logger.error(f"Failed to set opt enabled status to {enabled} for opt {opt_id}") + return False + return True + + async def cm_put_opt(self, version: int, folder: str, sequence: int, geki_ver: int, geki_seq: int, mai_ver: int, mai_seq: int) -> Optional[int]: + sql = insert(cm_opts).values( + version=version, + name=folder, + sequence=sequence, + gekiVersion=geki_ver, + gekiReleaseVer=geki_seq, + maiSequence=mai_ver, + maiReleaseVer=mai_seq, + ) + + conflict = sql.on_duplicate_key_update( + sequence=sequence, + gekiVersion=geki_ver, + gekiReleaseVer=geki_seq, + maiSequence=mai_ver, + maiReleaseVer=mai_seq, + whenRead=datetime.now() + ) + + result = await self.execute(conflict) + if result is None: + self.logger.warning(f"Failed to insert opt! version {version} folder {folder} sequence {sequence}") + return None + return result.lastrowid + + async def cm_get_opt_by_version_folder(self, version: int, folder: str) -> Optional[Row]: + result = await self.execute(cm_opts.select(and_( + opts.c.version == version, + opts.c.name == folder, + ))) + + if result is None: + return None + return result.fetchone() From ada9377c06823c5aeead2fbc8f6e695751f9da3c Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Wed, 9 Apr 2025 18:09:41 -0400 Subject: [PATCH 119/130] ongeki: remove BM card duplicate check --- titles/ongeki/read.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/titles/ongeki/read.py b/titles/ongeki/read.py index 4797dcb..435ebb1 100644 --- a/titles/ongeki/read.py +++ b/titles/ongeki/read.py @@ -74,18 +74,6 @@ class OngekiReader(BaseReader): troot = ET.fromstring(f.read()) card_id = int(troot.find("Name").find("id").text) - - # skip already existing cards - # Hay1tsme 2025/04/08: What is this for, and why does it only check for BM cards? - if ( - await self.data.static.get_card( - OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, card_id - ) - is not None - ): - self.logger.info(f"Card {card_id} already added, skipping") - continue - name = troot.find("Name").find("str").text chara_id = int(troot.find("CharaID").find("id").text) nick_name = troot.find("NickName").text From ce475e801b81fbd2ef3e57fd7101bc5a0b65d1fc Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 17 Apr 2025 20:11:33 -0400 Subject: [PATCH 120/130] allnet: save billing traces --- core/allnet.py | 125 +++++++++++++----- core/config.py | 8 +- .../27e3434740df_add_billing_tables.py | 66 +++++++++ core/data/schema/arcade.py | 118 ++++++++++++++++- example_config/core.yaml | 1 + 5 files changed, 280 insertions(+), 38 deletions(-) create mode 100644 core/data/alembic/versions/27e3434740df_add_billing_tables.py diff --git a/core/allnet.py b/core/allnet.py index c878870..2cb823e 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -586,31 +586,12 @@ class BillingServlet: rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read()) signer = PKCS1_v1_5.new(rsa) digest = SHA.new() - traces: List[TraceData] = [] try: req = BillingInfo(req_dict[0]) except KeyError as e: self.logger.error(f"Billing request failed to parse: {e}") return PlainTextResponse("result=5&linelimit=&message=field is missing or formatting is incorrect\r\n") - for x in range(1, len(req_dict)): - if not req_dict[x]: - continue - - try: - tmp = TraceData(req_dict[x]) - if tmp.trace_type == TraceDataType.CHARGE: - tmp = TraceDataCharge(req_dict[x]) - elif tmp.trace_type == TraceDataType.EVENT: - tmp = TraceDataEvent(req_dict[x]) - elif tmp.trace_type == TraceDataType.CREDIT: - tmp = TraceDataCredit(req_dict[x]) - - traces.append(tmp) - - except KeyError as e: - self.logger.warning(f"Tracelog failed to parse: {e}") - kc_serial_bytes = req.keychipid.encode() @@ -618,7 +599,7 @@ class BillingServlet: if machine is None and not self.config.server.allow_unregistered_serials: msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}." await self.data.base.log_event( - "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg, ip=request_ip, game=req.gameid, version=req.gamever + "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg, ip=request_ip, game=req.gameid, version=str(req.gamever) ) self.logger.warning(msg) @@ -629,18 +610,79 @@ class BillingServlet: "billing_type": req.billingtype.name, "nearfull": req.nearfull, "playlimit": req.playlimit, + "messages": [] } if machine is not None: - await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, "", log_details, None, machine['arcade'], machine['id'], request_ip, req.gameid, req.gamever) + for x in range(1, len(req_dict)): + if not req_dict[x]: + continue + + try: + tmp = TraceData(req_dict[x]) + if tmp.trace_type == TraceDataType.CHARGE: + tmp = TraceDataCharge(req_dict[x]) + if self.config.allnet.save_billing: + await self.data.arcade.billing_add_charge( + machine['id'], + tmp.game_id, + float(tmp.game_version), + tmp.play_count, + tmp.play_limit, + tmp.product_code, + tmp.product_count, + tmp.func_type, + tmp.player_number + ) + + self.logger.info( + f"Charge Trace from {req.keychipid}: {tmp.game_id} v{tmp.game_version} - player {tmp.player_number} got {tmp.product_count} of {tmp.product_code} func {tmp.func_type}" + ) + + elif tmp.trace_type == TraceDataType.EVENT: + tmp = TraceDataEvent(req_dict[x]) + log_details['messages'].append(tmp.message) + self.logger.info(f"Event Trace from {req.keychipid}: {tmp.message}") + + elif tmp.trace_type == TraceDataType.CREDIT: + tmp = TraceDataCredit(req_dict[x]) + if self.config.allnet.save_billing: + await self.data.arcade.billing_set_credit( + machine['id'], + tmp.chute_type.value, + tmp.service_type.value, + tmp.operation_type.value, + tmp.coin_rate0, + tmp.coin_rate1, + tmp.bonus_addition, + tmp.credit_rate, + tmp.credit0, + tmp.credit1, + tmp.credit2, + tmp.credit3, + tmp.credit4, + tmp.credit5, + tmp.credit6, + tmp.credit7 + ) + + self.logger.info( + f"Credit Trace from {req.keychipid}: {tmp.operation_type} mode, {tmp.credit_rate} coins per credit, Consumed {tmp.credit0} | {tmp.credit1} | {tmp.credit2} | {tmp.credit3} | {tmp.credit4} | {tmp.credit5} | {tmp.credit6} | {tmp.credit7} | " + ) + + except KeyError as e: + self.logger.warning(f"Tracelog failed to parse: {e}") + + await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, "", log_details, None, machine['arcade'], machine['id'], request_ip, req.gameid, str(req.gamever)) self.logger.info( f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount " f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}" ) + else: log_details['serial'] = req.keychipid - await self.data.base.log_event("billing", "BILLING_CHECKIN_OK_UNREG", logging.INFO, "", log_details, None, None, None, request_ip, req.gameid, req.gamever) + await self.data.base.log_event("billing", "BILLING_CHECKIN_OK_UNREG", logging.INFO, "", log_details, None, None, None, request_ip, req.gameid, str(req.gamever)) self.logger.info( f"Unregistered Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount " @@ -768,14 +810,27 @@ class BillingType(Enum): A = 1 B = 0 +class TraceDataCreditChuteType(Enum): + COMMON = 0 + INDIVIDUAL = 1 + +class TraceDataCreditOperationType(Enum): + COIN = 0 + FREEPLAY = 1 + class float5: - def __init__(self, n: str = "0") -> None: + def __init__(self, n: str = "0"): nf = float(n) if nf > 999.9 or nf < 0: raise ValueError('float5 must be between 0.000 and 999.9 inclusive') - - return nf + self.val = nf + def __float__(self) -> float: + return self.val + + def __str__(self) -> str: + return f"%.{2 - int(math.log10(self.val))+1}f" % self.val + @classmethod def to_str(cls, f: float): return f"%.{2 - int(math.log10(f))+1}f" % f @@ -786,13 +841,13 @@ class BillingInfo: self.keychipid = str(data.get("keychipid", None)) self.functype = int(data.get("functype", None)) self.gameid = str(data.get("gameid", None)) - self.gamever = float(data.get("gamever", None)) + self.gamever = float5(data.get("gamever", None)) self.boardid = str(data.get("boardid", None)) self.tenpoip = str(data.get("tenpoip", None)) - self.libalibver = float(data.get("libalibver", None)) + self.libalibver = float5(data.get("libalibver", None)) self.datamax = int(data.get("datamax", None)) self.billingtype = BillingType(int(data.get("billingtype", None))) - self.protocolver = float(data.get("protocolver", None)) + self.protocolver = float5(data.get("protocolver", None)) self.operatingfix = bool(data.get("operatingfix", None)) self.traceleft = int(data.get("traceleft", None)) self.requestno = int(data.get("requestno", None)) @@ -825,7 +880,7 @@ class TraceData: self.date = datetime.strptime(data.get("dt", None), BILLING_DT_FORMAT) self.keychip = str(data.get("kn", None)) - self.lib_ver = float(data.get("alib", 0)) + self.lib_ver = float5(data.get("alib", 0)) except Exception as e: raise KeyError(e) @@ -834,7 +889,7 @@ class TraceDataCharge(TraceData): super().__init__(data) try: self.game_id = str(data.get("gi", None)) # these seem optional...? - self.game_version = float(data.get("gv", 0)) + self.game_version = float5(data.get("gv", 0)) self.board_serial = str(data.get("bn", None)) self.shop_ip = str(data.get("ti", None)) self.play_count = int(data.get("pc", None)) @@ -858,9 +913,9 @@ class TraceDataCredit(TraceData): def __init__(self, data: Dict) -> None: super().__init__(data) try: - self.chute_type = int(data.get("cct", None)) - self.service_type = int(data.get("cst", None)) - self.operation_type = int(data.get("cop", None)) + self.chute_type = TraceDataCreditChuteType(int(data.get("cct", None))) + self.service_type = TraceDataCreditChuteType(int(data.get("cst", None))) + self.operation_type = TraceDataCreditOperationType(int(data.get("cop", None))) self.coin_rate0 = int(data.get("cr0", None)) self.coin_rate1 = int(data.get("cr1", None)) self.bonus_addition = int(data.get("cba", None)) @@ -884,7 +939,7 @@ class BillingResponse: nearfull: str = "", nearfull_sig: str = "", request_num: int = 1, - protocol_ver: float = 1.000, + protocol_ver: float5 = float5("1.000"), playhistory: str = "000000/0:000000/0:000000/0", ) -> None: self.result = 0 @@ -898,7 +953,7 @@ class BillingResponse: self.nearfull = nearfull self.nearfullsig = nearfull_sig self.linelimit = 100 - self.protocolver = float5.to_str(protocol_ver) + self.protocolver = str(protocol_ver) # playhistory -> YYYYMM/C:... # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period diff --git a/core/config.py b/core/config.py index e05323b..eb02c4e 100644 --- a/core/config.py +++ b/core/config.py @@ -362,7 +362,7 @@ class AllnetConfig: ) @property - def allow_online_updates(self) -> int: + def allow_online_updates(self) -> bool: return CoreConfig.get_config_field( self.__config, "core", "allnet", "allow_online_updates", default=False ) @@ -373,6 +373,12 @@ class AllnetConfig: self.__config, "core", "allnet", "update_cfg_folder", default="" ) + @property + def save_billing(self) -> bool: + return CoreConfig.get_config_field( + self.__config, "core", "allnet", "save_billing", default=False + ) + class BillingConfig: def __init__(self, parent_config: "CoreConfig") -> None: self.__config = parent_config diff --git a/core/data/alembic/versions/27e3434740df_add_billing_tables.py b/core/data/alembic/versions/27e3434740df_add_billing_tables.py new file mode 100644 index 0000000..191336b --- /dev/null +++ b/core/data/alembic/versions/27e3434740df_add_billing_tables.py @@ -0,0 +1,66 @@ +"""add_billing_tables + +Revision ID: 27e3434740df +Revises: ae364c078429 +Create Date: 2025-04-17 18:32:06.008601 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '27e3434740df' +down_revision = 'ae364c078429' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('machine_billing_charge', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('machine', sa.Integer(), nullable=False), + sa.Column('game_id', sa.CHAR(length=5), nullable=False), + sa.Column('game_ver', sa.FLOAT(), nullable=False), + sa.Column('play_count', sa.INTEGER(), nullable=False), + sa.Column('play_limit', sa.INTEGER(), nullable=False), + sa.Column('product_code', sa.INTEGER(), nullable=False), + sa.Column('product_count', sa.INTEGER(), nullable=False), + sa.Column('func_type', sa.INTEGER(), nullable=False), + sa.Column('player_number', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint(['machine'], ['machine.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + mysql_charset='utf8mb4' + ) + op.create_table('machine_billing_credit', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('machine', sa.Integer(), nullable=False), + sa.Column('chute_type', sa.INTEGER(), nullable=False), + sa.Column('service_type', sa.INTEGER(), nullable=False), + sa.Column('operation_type', sa.INTEGER(), nullable=False), + sa.Column('coin_rate0', sa.INTEGER(), nullable=False), + sa.Column('coin_rate1', sa.INTEGER(), nullable=False), + sa.Column('coin_bonus', sa.INTEGER(), nullable=False), + sa.Column('credit_rate', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot0', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot1', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot2', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot3', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot4', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot5', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot6', sa.INTEGER(), nullable=False), + sa.Column('coin_count_slot7', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint(['machine'], ['machine.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('machine'), + mysql_charset='utf8mb4' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('machine_billing_credit') + op.drop_table('machine_billing_charge') + # ### end Alembic commands ### diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index 038077d..cfbff96 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row from sqlalchemy.sql import func, select from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint -from sqlalchemy.types import JSON, Boolean, Integer, String +from sqlalchemy.types import JSON, Boolean, Integer, String, BIGINT, INTEGER, CHAR, FLOAT from core.data.schema.base import BaseData, metadata @@ -67,6 +67,56 @@ arcade_owner: Table = Table( mysql_charset="utf8mb4", ) +billing_charge: Table = Table( + "machine_billing_charge", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column( + "machine", + Integer, + ForeignKey("machine.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("game_id", CHAR(5), nullable=False), + Column("game_ver", FLOAT, nullable=False), + Column("play_count", INTEGER, nullable=False), + Column("play_limit", INTEGER, nullable=False), + Column("product_code", INTEGER, nullable=False), + Column("product_count", INTEGER, nullable=False), + Column("func_type", INTEGER, nullable=False), + Column("player_number", INTEGER, nullable=False), + mysql_charset="utf8mb4", +) + +# These settings are only really of interest +# for real cabinets operating as pay-to-play +billing_credit: Table = Table( + "machine_billing_credit", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column( + "machine", + Integer, + ForeignKey("machine.id", ondelete="cascade", onupdate="cascade"), + nullable=False, unique=True + ), + Column("chute_type", INTEGER, nullable=False), + Column("service_type", INTEGER, nullable=False), + Column("operation_type", INTEGER, nullable=False), + Column("coin_rate0", INTEGER, nullable=False), + Column("coin_rate1", INTEGER, nullable=False), + Column("coin_bonus", INTEGER, nullable=False), + Column("credit_rate", INTEGER, nullable=False), + Column("coin_count_slot0", INTEGER, nullable=False), + Column("coin_count_slot1", INTEGER, nullable=False), + Column("coin_count_slot2", INTEGER, nullable=False), + Column("coin_count_slot3", INTEGER, nullable=False), + Column("coin_count_slot4", INTEGER, nullable=False), + Column("coin_count_slot5", INTEGER, nullable=False), + Column("coin_count_slot6", INTEGER, nullable=False), + Column("coin_count_slot7", INTEGER, nullable=False), + mysql_charset="utf8mb4", +) class ArcadeData(BaseData): async def get_machine(self, serial: Optional[str] = None, id: Optional[int] = None) -> Optional[Row]: @@ -345,6 +395,71 @@ class ArcadeData(BaseData): return result.fetchone()['count_1'] self.logger.error("Failed to count machine serials that start with A69A!") + async def billing_add_charge(self, machine_id: int, game_id: str, game_ver: float, playcount: int, playlimit, product_code: int, product_count: int, func_type: int, player_num: int) -> Optional[int]: + result = await self.execute(billing_charge.insert().values( + machine=machine_id, + game_id=game_id, + game_ver=game_ver, + play_count=playcount, + play_limit=playlimit, + product_code=product_code, + product_count=product_count, + func_type=func_type, + player_num=player_num + )) + + if result is None: + self.logger.error(f"Failed to add billing charge for machine {machine_id}!") + return None + return result.lastrowid + + async def billing_set_credit(self, machine_id: int, chute_type: int, service_type: int, op_mode: int, coin_rate0: int, coin_rate1: int, + bonus_adder: int, coin_to_credit_rate: int, coin_count_slot0: int, coin_count_slot1: int, coin_count_slot2: int, coin_count_slot3: int, + coin_count_slot4: int, coin_count_slot5: int, coin_count_slot6: int, coin_count_slot7: int) -> Optional[int]: + + sql = insert(billing_credit).values( + machine=machine_id, + chute_type=chute_type, + service_type=service_type, + operation_type=op_mode, + coin_rate0=coin_rate0, + coin_rate1=coin_rate1, + coin_bonus=bonus_adder, + credit_rate=coin_to_credit_rate, + coin_count_slot0=coin_count_slot0, + coin_count_slot1=coin_count_slot1, + coin_count_slot2=coin_count_slot2, + coin_count_slot3=coin_count_slot3, + coin_count_slot4=coin_count_slot4, + coin_count_slot5=coin_count_slot5, + coin_count_slot6=coin_count_slot6, + coin_count_slot7=coin_count_slot7, + ) + + conflict = sql.on_duplicate_key_update( + chute_type=chute_type, + service_type=service_type, + operation_type=op_mode, + coin_rate0=coin_rate0, + coin_rate1=coin_rate1, + coin_bonus=bonus_adder, + credit_rate=coin_to_credit_rate, + coin_count_slot0=coin_count_slot0, + coin_count_slot1=coin_count_slot1, + coin_count_slot2=coin_count_slot2, + coin_count_slot3=coin_count_slot3, + coin_count_slot4=coin_count_slot4, + coin_count_slot5=coin_count_slot5, + coin_count_slot6=coin_count_slot6, + coin_count_slot7=coin_count_slot7, + ) + + result = await self.execute(conflict) + if result is None: + self.logger.error(f"Failed to set billing credit settings for machine {machine_id}!") + return None + return result.lastrowid + def format_serial( self, platform_code: str, platform_rev: int, serial_letter: str, serial_num: int, append: int, dash: bool = False ) -> str: @@ -371,7 +486,6 @@ class ArcadeData(BaseData): month = ((month - 1) + 9) % 12 # Offset so April=0 return f"{year:02}{month // 6:01}{month % 6 + 1:01}" - def parse_keychip_suffix(self, suffix: str) -> tuple[int, int]: year = int(suffix[0:2]) half = int(suffix[2]) diff --git a/example_config/core.yaml b/example_config/core.yaml index 0f047f0..fa04a67 100644 --- a/example_config/core.yaml +++ b/example_config/core.yaml @@ -45,6 +45,7 @@ allnet: loglevel: "info" allow_online_updates: False update_cfg_folder: "" + save_billing: True billing: standalone: True From eea9ca21ca83505b9dd71a9e8708c50adce0b834 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 24 Apr 2025 23:04:01 -0400 Subject: [PATCH 121/130] allnet: basic playhistory --- core/allnet.py | 32 ++++++++-- .../f6007bbf057d_add_billing_playcount.py | 50 ++++++++++++++++ core/data/schema/arcade.py | 58 ++++++++++++++++++- 3 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 core/data/alembic/versions/f6007bbf057d_add_billing_playcount.py diff --git a/core/allnet.py b/core/allnet.py index 2cb823e..cd349f7 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -593,7 +593,6 @@ class BillingServlet: return PlainTextResponse("result=5&linelimit=&message=field is missing or formatting is incorrect\r\n") kc_serial_bytes = req.keychipid.encode() - machine = await self.data.arcade.get_machine(req.keychipid) if machine is None and not self.config.server.allow_unregistered_serials: @@ -614,6 +613,32 @@ class BillingServlet: } if machine is not None: + if self.config.allnet.save_billing: + lastcredit = await self.data.arcade.billing_get_credit(machine['id'], req.gameid) + if lastcredit is not None: + last_playct = lastcredit['playcount'] + else: + last_playct = 0 + + # Technically if a cab resets it's playcount and then does more plays then the previous + # playcount before a billing checkin occours, we will lose plays equal to the current playcount. + if req.playcnt < last_playct: await self.data.arcade.billing_add_playcount(machine['id'], req.gameid, req.playcnt) + elif req.playcnt == last_playct: pass # No plays since last checkin, skip update + else: await self.data.arcade.billing_add_playcount(machine['id'], req.gameid, req.playcnt - last_playct) + + plays = await self.data.arcade.billing_get_playcount_3mo(machine['id'], req.gameid) + if plays is not None and len(plays) > 0: + playhist = "" + + for x in range(len(plays), 0, -1): playhist += f"{plays[x]['year']:04d}{plays[x]['month']:02d}/{plays[x]['playct']}:" + playhist = playhist[:-1] + + else: + playhist = "000000/0:000000/0:000000/0" + + else: + playhist = "000000/0:000000/0:000000/0" + for x in range(1, len(req_dict)): if not req_dict[x]: continue @@ -649,6 +674,7 @@ class BillingServlet: if self.config.allnet.save_billing: await self.data.arcade.billing_set_credit( machine['id'], + req.gameid, tmp.chute_type.value, tmp.service_type.value, tmp.operation_type.value, @@ -708,9 +734,7 @@ class BillingServlet: digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes) nearfull_sig = signer.sign(digest).hex() - # TODO: playhistory - - resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver) + resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver, playhist) resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n" diff --git a/core/data/alembic/versions/f6007bbf057d_add_billing_playcount.py b/core/data/alembic/versions/f6007bbf057d_add_billing_playcount.py new file mode 100644 index 0000000..e6a5259 --- /dev/null +++ b/core/data/alembic/versions/f6007bbf057d_add_billing_playcount.py @@ -0,0 +1,50 @@ +"""add_billing_playcount + +Revision ID: f6007bbf057d +Revises: 27e3434740df +Create Date: 2025-04-19 18:20:35.554137 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'f6007bbf057d' +down_revision = '27e3434740df' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('machine_billing_playcount', + sa.Column('id', sa.BIGINT(), nullable=False), + sa.Column('machine', sa.Integer(), nullable=False), + sa.Column('game_id', sa.CHAR(length=5), nullable=False), + sa.Column('year', sa.INTEGER(), nullable=False), + sa.Column('month', sa.INTEGER(), nullable=False), + sa.Column('playct', sa.BIGINT(), server_default='1', nullable=False), + sa.ForeignKeyConstraint(['machine'], ['machine.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('machine'), + sa.UniqueConstraint('machine', 'game_id', 'year', 'month', name='machine_billing_playcount_uk'), + mysql_charset='utf8mb4' + ) + op.add_column('machine_billing_credit', sa.Column('game_id', sa.CHAR(length=5), nullable=False)) + op.drop_constraint("machine_billing_credit_ibfk_1", "machine_billing_credit", "foreignkey") + op.drop_index('machine', table_name='machine_billing_credit') + op.create_unique_constraint('machine_billing_credit_uk', 'machine_billing_credit', ['machine', 'game_id']) + op.create_foreign_key("machine_billing_credit_ibfk_1", "machine_billing_credit", "machine", ["machine"], ["id"], onupdate='cascade', ondelete='cascade') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("machine_billing_credit_ibfk_1", "machine_billing_credit", "foreignkey") + op.drop_constraint('machine_billing_credit_uk', 'machine_billing_credit', type_='unique') + op.create_index('machine', 'machine_billing_credit', ['machine'], unique=True) + op.create_foreign_key("machine_billing_credit_ibfk_1", "machine_billing_credit", "machine", ["machine"], ["id"], onupdate='cascade', ondelete='cascade') + op.drop_column('machine_billing_credit', 'game_id') + op.drop_table('machine_billing_playcount') + # ### end Alembic commands ### diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index cfbff96..2053f62 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -1,7 +1,8 @@ import re from typing import List, Optional +from datetime import datetime -from sqlalchemy import Column, Table, and_, or_ +from sqlalchemy import Column, Table, and_, or_, UniqueConstraint from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Row from sqlalchemy.sql import func, select @@ -98,8 +99,9 @@ billing_credit: Table = Table( "machine", Integer, ForeignKey("machine.id", ondelete="cascade", onupdate="cascade"), - nullable=False, unique=True + nullable=False ), + Column("game_id", CHAR(5), nullable=False), Column("chute_type", INTEGER, nullable=False), Column("service_type", INTEGER, nullable=False), Column("operation_type", INTEGER, nullable=False), @@ -115,6 +117,25 @@ billing_credit: Table = Table( Column("coin_count_slot5", INTEGER, nullable=False), Column("coin_count_slot6", INTEGER, nullable=False), Column("coin_count_slot7", INTEGER, nullable=False), + UniqueConstraint("machine", "game_id", name="machine_billing_credit_uk"), + mysql_charset="utf8mb4", +) + +billing_playct: Table = Table( + "machine_billing_playcount", + metadata, + Column("id", BIGINT, primary_key=True, nullable=False), + Column( + "machine", + Integer, + ForeignKey("machine.id", ondelete="cascade", onupdate="cascade"), + nullable=False, unique=True + ), + Column("game_id", CHAR(5), nullable=False), + Column("year", INTEGER, nullable=False), + Column("month", INTEGER, nullable=False), + Column("playct", BIGINT, nullable=False, server_default="1"), + UniqueConstraint("machine", "game_id", "year", "month", name="machine_billing_playcount_uk"), mysql_charset="utf8mb4", ) @@ -413,12 +434,13 @@ class ArcadeData(BaseData): return None return result.lastrowid - async def billing_set_credit(self, machine_id: int, chute_type: int, service_type: int, op_mode: int, coin_rate0: int, coin_rate1: int, + async def billing_set_credit(self, machine_id: int, game_id: str, chute_type: int, service_type: int, op_mode: int, coin_rate0: int, coin_rate1: int, bonus_adder: int, coin_to_credit_rate: int, coin_count_slot0: int, coin_count_slot1: int, coin_count_slot2: int, coin_count_slot3: int, coin_count_slot4: int, coin_count_slot5: int, coin_count_slot6: int, coin_count_slot7: int) -> Optional[int]: sql = insert(billing_credit).values( machine=machine_id, + game_id=game_id, chute_type=chute_type, service_type=service_type, operation_type=op_mode, @@ -460,6 +482,36 @@ class ArcadeData(BaseData): return None return result.lastrowid + async def billing_get_credit(self, machine_id: int, game_id: str) -> Optional[Row]: + result = await self.execute(billing_credit.select(billing_credit.c.machine == machine_id)) + if result: + return result.fetchone() + + async def billing_add_playcount(self, machine_id: int, game_id: str, playct: int = 1) -> None: + now = datetime.now() + sql = insert(billing_playct).values( + machine=machine_id, + game_id=game_id, + year=now.year, + month=now.month, + playct=playct + ) + + conflict = sql.on_duplicate_key_update(playct=billing_playct.c.playct + playct) + result = await self.execute(conflict) + + if result is None: + self.logger.error(f"Failed to add playcount for machine {machine_id} running {game_id}") + + async def billing_get_playcount_3mo(self, machine_id: int, game_id: str) -> Optional[List[Row]]: + result = await self.execute(billing_playct.select(and_( + billing_playct.c.machine == machine_id, + billing_playct.c.game_id == game_id + )).order_by(billing_playct.c.year.desc(), billing_playct.c.month.desc()).limit(3)) + + if result is not None: + return result.fetchall() + def format_serial( self, platform_code: str, platform_rev: int, serial_letter: str, serial_num: int, append: int, dash: bool = False ) -> str: From a74ca853000116b91e636d84df0892462d19c97b Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Thu, 24 Apr 2025 23:56:19 -0400 Subject: [PATCH 122/130] allnet: fix playhistory --- core/allnet.py | 15 +++++---------- core/data/schema/arcade.py | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index cd349f7..b062541 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -611,12 +611,13 @@ class BillingServlet: "playlimit": req.playlimit, "messages": [] } + playhist = "000000/0:000000/0:000000/0" if machine is not None: if self.config.allnet.save_billing: - lastcredit = await self.data.arcade.billing_get_credit(machine['id'], req.gameid) + lastcredit = await self.data.arcade.billing_get_last_playcount(machine['id'], req.gameid) if lastcredit is not None: - last_playct = lastcredit['playcount'] + last_playct = lastcredit['playct'] else: last_playct = 0 @@ -630,14 +631,8 @@ class BillingServlet: if plays is not None and len(plays) > 0: playhist = "" - for x in range(len(plays), 0, -1): playhist += f"{plays[x]['year']:04d}{plays[x]['month']:02d}/{plays[x]['playct']}:" + for x in range(len(plays) - 1, -1, -1): playhist += f"{plays[x]['year']:04d}{plays[x]['month']:02d}/{plays[x]['playct']}:" playhist = playhist[:-1] - - else: - playhist = "000000/0:000000/0:000000/0" - - else: - playhist = "000000/0:000000/0:000000/0" for x in range(1, len(req_dict)): if not req_dict[x]: @@ -693,7 +688,7 @@ class BillingServlet: ) self.logger.info( - f"Credit Trace from {req.keychipid}: {tmp.operation_type} mode, {tmp.credit_rate} coins per credit, Consumed {tmp.credit0} | {tmp.credit1} | {tmp.credit2} | {tmp.credit3} | {tmp.credit4} | {tmp.credit5} | {tmp.credit6} | {tmp.credit7} | " + f"Credit Trace from {req.keychipid}: {tmp.operation_type} mode, {tmp.credit_rate} coins per credit, breakdown: {tmp.credit0} | {tmp.credit1} | {tmp.credit2} | {tmp.credit3} | {tmp.credit4} | {tmp.credit5} | {tmp.credit6} | {tmp.credit7} | " ) except KeyError as e: diff --git a/core/data/schema/arcade.py b/core/data/schema/arcade.py index 2053f62..d1790b8 100644 --- a/core/data/schema/arcade.py +++ b/core/data/schema/arcade.py @@ -426,13 +426,20 @@ class ArcadeData(BaseData): product_code=product_code, product_count=product_count, func_type=func_type, - player_num=player_num + player_number=player_num )) if result is None: self.logger.error(f"Failed to add billing charge for machine {machine_id}!") return None return result.lastrowid + + async def billing_get_last_charge(self, machine_id: int, game_id: str) -> Optional[Row]: + result = await self.execute(billing_charge.select( + and_(billing_charge.c.machine == machine_id, billing_charge.c.game_id == game_id) + ).order_by(billing_charge.c.id.desc()).limit(3)) + if result: + return result.fetchone() async def billing_set_credit(self, machine_id: int, game_id: str, chute_type: int, service_type: int, op_mode: int, coin_rate0: int, coin_rate1: int, bonus_adder: int, coin_to_credit_rate: int, coin_count_slot0: int, coin_count_slot1: int, coin_count_slot2: int, coin_count_slot3: int, @@ -483,7 +490,9 @@ class ArcadeData(BaseData): return result.lastrowid async def billing_get_credit(self, machine_id: int, game_id: str) -> Optional[Row]: - result = await self.execute(billing_credit.select(billing_credit.c.machine == machine_id)) + result = await self.execute(billing_credit.select( + and_(billing_credit.c.machine == machine_id, billing_credit.c.game_id == game_id) + )) if result: return result.fetchone() @@ -512,6 +521,15 @@ class ArcadeData(BaseData): if result is not None: return result.fetchall() + async def billing_get_last_playcount(self, machine_id: int, game_id: str) -> Optional[Row]: + result = await self.execute(billing_playct.select(and_( + billing_playct.c.machine == machine_id, + billing_playct.c.game_id == game_id + )).order_by(billing_playct.c.year.desc(), billing_playct.c.month.desc()).limit(1)) + + if result is not None: + return result.fetchone() + def format_serial( self, platform_code: str, platform_rev: int, serial_letter: str, serial_num: int, append: int, dash: bool = False ) -> str: From f1a0557f94e64bcbe6c5b38ff89d8f237c81b272 Mon Sep 17 00:00:00 2001 From: SoulGateKey Date: Thu, 1 May 2025 17:30:06 +0000 Subject: [PATCH 123/130] fix mai2 internal ver --- titles/mai2/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/titles/mai2/index.py b/titles/mai2/index.py index 86923e7..d8e2a4f 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -309,7 +309,7 @@ class Mai2Servlet(BaseServlet): elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES elif version >= 145 and version <150: # BUDDiES PLUS - internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS, + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS elif version >=150: internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM @@ -333,7 +333,7 @@ class Mai2Servlet(BaseServlet): elif version >= 140 and version < 145: # BUDDiES internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES elif version >= 145 and version <150: # BUDDiES PLUS - internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS, + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES_PLUS elif version >=150: internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM From e6d78886551491d11a969203ed620f05c7267557 Mon Sep 17 00:00:00 2001 From: Hay1tsme Date: Sat, 3 May 2025 15:51:50 -0400 Subject: [PATCH 124/130] billing: fix infinite loop --- core/allnet.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index b062541..0912f56 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -712,15 +712,12 @@ class BillingServlet: if req.traceleft > 0: self.logger.warning(f"{req.traceleft} unsent tracelogs") - kc_playlimit = req.playlimit - kc_nearfull = req.nearfull - while req.playcnt > req.playlimit: - kc_playlimit += 1024 - kc_nearfull += 1024 + playlimit = req.playlimit + while req.playcnt > playlimit: + playlimit += 1024 - playlimit = kc_playlimit - nearfull = kc_nearfull + (req.billingtype.value * 0x00010000) + nearfull = req.nearfull + (req.billingtype.value * 0x00010000) digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes) playlimit_sig = signer.sign(digest).hex() @@ -734,7 +731,7 @@ class BillingServlet: resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n" self.logger.debug(f"response {vars(resp)}") - if req.traceleft > 0: + if req.traceleft > 0: # TODO: should probably move this up so we don't do a ton of work that doesn't get used self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs") return PlainTextResponse("result=6&waittime=0&linelimit=20\r\n") From 5e5365d22b817e08d496201540266b5044754b48 Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Sat, 31 May 2025 07:15:35 +1200 Subject: [PATCH 125/130] Allnet Lite Power On Support --- core/allnet.py | 106 +++++++++++++++++++++++++++++++++++---- core/app.py | 1 + core/config.py | 5 ++ example_config/core.yaml | 1 + 4 files changed, 103 insertions(+), 10 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 0912f56..2115b9a 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -7,6 +7,7 @@ import logging import coloredlogs import urllib.parse import math +import random from typing import Dict, List, Any, Optional, Union, Final from logging.handlers import TimedRotatingFileHandler from starlette.requests import Request @@ -17,7 +18,10 @@ from datetime import datetime from enum import Enum from Crypto.PublicKey import RSA from Crypto.Hash import SHA +from Crypto.Cipher import AES +from Crypto.Util.Padding import pad from Crypto.Signature import PKCS1_v1_5 +import os from os import path, environ, mkdir, access, W_OK from .config import CoreConfig @@ -132,12 +136,20 @@ class AllnetServlet: async def handle_poweron(self, request: Request): request_ip = Utils.get_ip_addr(request) pragma_header = request.headers.get('Pragma', "") + useragent_header = request.headers.get('User-Agent', "") is_dfi = pragma_header == "DFI" + is_lite = useragent_header[5:] == "Windows/Lite" data = await request.body() + + if not self.config.allnet.allnet_lite_key and is_lite: + self.logger.error("!!!LITE KEY NOT SET!!!") + os._exit(1) try: if is_dfi: req_urlencode = self.from_dfi(data) + elif is_lite: + req_urlencode = self.dec_lite(self.config.allnet.allnet_lite_key, data[:16], data) else: req_urlencode = data @@ -145,20 +157,30 @@ class AllnetServlet: if req_dict is None: raise AllnetRequestException() - req = AllnetPowerOnRequest(req_dict[0]) + if is_lite: + req = AllnetPowerOnRequestLite(req_dict[0]) + else: + req = AllnetPowerOnRequest(req_dict[0]) # Validate the request. Currently we only validate the fields we plan on using - if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver: + if not req.game_id or not req.ver or not req.serial or not req.token and is_lite: raise AllnetRequestException( f"Bad auth request params from {request_ip} - {vars(req)}" ) + elif not is_lite: + if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver: + raise AllnetRequestException( + f"Bad auth request params from {request_ip} - {vars(req)}" + ) except AllnetRequestException as e: if e.message != "": self.logger.error(e) return PlainTextResponse() - if req.format_ver == 3: + if is_lite: + resp = AllnetPowerOnResponseLite(req.token) + elif req.format_ver == 3: resp = AllnetPowerOnResponse3(req.token) elif req.format_ver == 2: resp = AllnetPowerOnResponse2() @@ -175,11 +197,14 @@ class AllnetServlet: ) self.logger.warning(msg) - resp.stat = ALLNET_STAT.bad_machine.value + if is_lite: + resp.result = ALLNET_STAT.bad_machine.value + else: + resp.stat = ALLNET_STAT.bad_machine.value resp_dict = {k: v for k, v in vars(resp).items() if v is not None} return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n") - if machine is not None: + if machine is not None and not is_lite: arcade = await self.data.arcade.get_arcade(machine["arcade"]) if self.config.server.check_arcade_ip: if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip: @@ -257,7 +282,10 @@ class AllnetServlet: ) self.logger.warning(msg) - resp.stat = ALLNET_STAT.bad_game.value + if is_lite: + resp.result = ALLNET_STAT.bad_game.value + else: + resp.stat = ALLNET_STAT.bad_game.value resp_dict = {k: v for k, v in vars(resp).items() if v is not None} return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n") @@ -265,8 +293,12 @@ class AllnetServlet: self.logger.info( f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}" ) - resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/" - resp.host = f"{self.config.server.hostname}:{self.config.server.port}" + if is_lite: + resp.uri1 = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/" + resp.uri2 = f"{self.config.server.hostname}:{self.config.server.port}" + else: + resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/" + resp.host = f"{self.config.server.hostname}:{self.config.server.port}" resp_dict = {k: v for k, v in vars(resp).items() if v is not None} resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) @@ -277,10 +309,16 @@ class AllnetServlet: int_ver = req.ver.replace(".", "") try: - resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial) + if is_lite: + resp.uri1, resp.uri2 = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial) + else: + resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial) except Exception as e: self.logger.error(f"Error running get_allnet_info for {req.game_id} - {e}") - resp.stat = ALLNET_STAT.bad_game.value + if is_lite: + resp.result = ALLNET_STAT.bad_game.value + else: + resp.stat = ALLNET_STAT.bad_game.value resp_dict = {k: v for k, v in vars(resp).items() if v is not None} return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n") @@ -308,6 +346,9 @@ class AllnetServlet: "Pragma": "DFI", }, ) + elif is_lite: + iv = bytes([random.randint(2, 255) for _ in range(16)]) + return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp_str)) return PlainTextResponse(resp_str) @@ -517,6 +558,17 @@ class AllnetServlet: zipped = zlib.compress(unzipped) return base64.b64encode(zipped) + def dec_lite(self, key, iv, data): + cipher = AES.new(bytes(key), AES.MODE_CBC, iv) + decrypted = cipher.decrypt(data) + return decrypted[16:].decode("utf-8") + + def enc_lite(self, key, iv, data): + decrypted = pad(bytes([0] * 16) + data.encode('utf-8'), 16) + cipher = AES.new(bytes(key), AES.MODE_CBC, iv) + encrypted = cipher.encrypt(decrypted) + return encrypted + class BillingServlet: def __init__(self, core_cfg: CoreConfig, cfg_folder: str) -> None: self.config = core_cfg @@ -773,6 +825,15 @@ class AllnetPowerOnResponse: self.minute = datetime.now().minute self.second = datetime.now().second +class AllnetPowerOnRequestLite: + def __init__(self, req: Dict) -> None: + if req is None: + raise AllnetRequestException("Request processing failed") + self.game_id: str = req.get("title_id", None) + self.ver: str = req.get("title_ver", None) + self.serial: str = req.get("client_id", None) + self.token: str = req.get("token", None) + class AllnetPowerOnResponse3(AllnetPowerOnResponse): def __init__(self, token) -> None: super().__init__() @@ -804,6 +865,30 @@ class AllnetPowerOnResponse2(AllnetPowerOnResponse): self.timezone = "+09:00" self.res_class = "PowerOnResponseV2" +class AllnetPowerOnResponseLite: + def __init__(self, token) -> None: + # Custom Allnet Lite response + self.result = 1 + self.place_id = "0123" + self.uri1 = "" + self.uri2 = "" + self.name = "ARTEMiS" + self.nickname = "ARTEMiS" + self.setting = "1" + self.region0 = "1" + self.region_name0 = "W" + self.region_name1 = "" + self.region_name2 = "" + self.region_name3 = "" + self.country = "CHN" + self.location_type = "1" + self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + self.client_timezone = "+0800" + self.res_ver = "3" + self.token = token + class AllnetDownloadOrderRequest: def __init__(self, req: Dict) -> None: self.game_id = req.get("game_id", "") @@ -1068,6 +1153,7 @@ app_billing = Starlette( allnet = AllnetServlet(cfg, cfg_dir) route_lst = [ Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]), + Route("/net/initialize", allnet.handle_poweron, methods=["GET", "POST"]), Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]), Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]), Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]), diff --git a/core/app.py b/core/app.py index fa1c8f2..b450b0c 100644 --- a/core/app.py +++ b/core/app.py @@ -75,6 +75,7 @@ if not cfg.allnet.standalone: allnet = AllnetServlet(cfg, cfg_dir) route_lst += [ Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]), + Route("/net/initialize", allnet.handle_poweron, methods=["GET", "POST"]), Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]), Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]), Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]), diff --git a/core/config.py b/core/config.py index eb02c4e..7b6833d 100644 --- a/core/config.py +++ b/core/config.py @@ -378,6 +378,11 @@ class AllnetConfig: return CoreConfig.get_config_field( self.__config, "core", "allnet", "save_billing", default=False ) + @property + def allnet_lite_key(self) -> bool: + return CoreConfig.get_config_field( + self.__config, "core", "allnet", "allnet_lite_key", default=[] + ) class BillingConfig: def __init__(self, parent_config: "CoreConfig") -> None: diff --git a/example_config/core.yaml b/example_config/core.yaml index fa04a67..53063e3 100644 --- a/example_config/core.yaml +++ b/example_config/core.yaml @@ -46,6 +46,7 @@ allnet: allow_online_updates: False update_cfg_folder: "" save_billing: True + allnet_lite_key: [] billing: standalone: True From 1e4cb0b380ec1993dc0f95b7a3d6a01ddbbe484f Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Sat, 31 May 2025 08:44:51 +1200 Subject: [PATCH 126/130] Simple Download Order implementation for Allnet Lite --- core/allnet.py | 40 ++++++++++++++++++++++++++++++++++++++-- core/app.py | 1 + 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 2115b9a..96c5631 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -355,12 +355,20 @@ class AllnetServlet: async def handle_dlorder(self, request: Request): request_ip = Utils.get_ip_addr(request) pragma_header = request.headers.get('Pragma', "") + useragent_header = request.headers.get('User-Agent', "") is_dfi = pragma_header == "DFI" + is_lite = useragent_header[5:] == "Windows/Lite" data = await request.body() + if not self.config.allnet.allnet_lite_key and is_lite: + self.logger.error("!!!LITE KEY NOT SET!!!") + os._exit(1) + try: if is_dfi: req_urlencode = self.from_dfi(data) + elif is_lite: + req_urlencode = self.dec_lite(self.config.allnet.allnet_lite_key, data[:16], data) else: req_urlencode = data.decode() @@ -368,7 +376,10 @@ class AllnetServlet: if req_dict is None: raise AllnetRequestException() - req = AllnetDownloadOrderRequest(req_dict[0]) + if is_lite: + req = AllnetDownloadOrderRequestLite(req_dict[0]) + else: + req = AllnetDownloadOrderRequest(req_dict[0]) # Validate the request. Currently we only validate the fields we plan on using if not req.game_id or not req.ver or not req.serial: @@ -384,7 +395,11 @@ class AllnetServlet: self.logger.info( f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}" ) - resp = AllnetDownloadOrderResponse(serial=req.serial) + + if is_lite: + resp = AllnetDownloadOrderResponseLite() + else: + resp = AllnetDownloadOrderResponse(serial=req.serial) if ( not self.config.allnet.allow_online_updates @@ -395,6 +410,9 @@ class AllnetServlet: return PlainTextResponse( self.to_dfi(resp) + b"\r\n", headers={ "Pragma": "DFI" } ) + elif is_lite: + iv = bytes([random.randint(2, 255) for _ in range(16)]) + return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp)) return PlainTextResponse(resp) else: @@ -405,6 +423,9 @@ class AllnetServlet: return PlainTextResponse( self.to_dfi(resp) + b"\r\n", headers={ "Pragma": "DFI" } ) + elif is_lite: + iv = bytes([random.randint(2, 255) for _ in range(16)]) + return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp)) return PlainTextResponse(resp) if path.exists( @@ -434,6 +455,9 @@ class AllnetServlet: "Pragma": "DFI", }, ) + elif is_lite: + iv = bytes([random.randint(2, 255) for _ in range(16)]) + return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, res_str)) return PlainTextResponse(res_str) @@ -896,12 +920,23 @@ class AllnetDownloadOrderRequest: self.serial = req.get("serial", "") self.encode = req.get("encode", "") +class AllnetDownloadOrderRequestLite: + def __init__(self, req: Dict) -> None: + self.game_id = req.get("title_id", "") + self.ver = req.get("title_ver", "") + self.serial = req.get("client_id", "") + class AllnetDownloadOrderResponse: def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None: self.stat = stat self.serial = serial self.uri = uri +class AllnetDownloadOrderResponseLite: + def __init__(self, result: int = 1, uri: str = "null") -> None: + self.result = result + self.uri = uri + class TraceDataType(Enum): CHARGE = 0 EVENT = 1 @@ -1155,6 +1190,7 @@ route_lst = [ Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]), Route("/net/initialize", allnet.handle_poweron, methods=["GET", "POST"]), Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]), + Route("/net/delivery/instruction", allnet.handle_dlorder, methods=["GET", "POST"]), Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]), Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]), Route("/naomitest.html", allnet.handle_naomitest), diff --git a/core/app.py b/core/app.py index b450b0c..ecae1bf 100644 --- a/core/app.py +++ b/core/app.py @@ -77,6 +77,7 @@ if not cfg.allnet.standalone: Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]), Route("/net/initialize", allnet.handle_poweron, methods=["GET", "POST"]), Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]), + Route("/net/delivery/instruction", allnet.handle_dlorder, methods=["GET", "POST"]), Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]), Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]), Route("/naomitest.html", allnet.handle_naomitest), From 3e848d684fd7da6195234e6c7a2b95474b98a5bf Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Sat, 31 May 2025 09:36:53 +1200 Subject: [PATCH 127/130] SDHJ title server support added + encryption --- titles/chuni/__init__.py | 2 +- titles/chuni/const.py | 1 + titles/chuni/index.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/titles/chuni/__init__.py b/titles/chuni/__init__.py index 226594a..faacc0f 100644 --- a/titles/chuni/__init__.py +++ b/titles/chuni/__init__.py @@ -8,4 +8,4 @@ index = ChuniServlet database = ChuniData reader = ChuniReader frontend = ChuniFrontend -game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT] +game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT, ChuniConstants.GAME_CODE_CHN] diff --git a/titles/chuni/const.py b/titles/chuni/const.py index fd05003..7c534d3 100644 --- a/titles/chuni/const.py +++ b/titles/chuni/const.py @@ -6,6 +6,7 @@ class ChuniConstants: GAME_CODE = "SDBT" GAME_CODE_NEW = "SDHD" GAME_CODE_INT = "SDGS" + GAME_CODE_CHN = "SDHJ" CONFIG_NAME = "chuni.yaml" diff --git a/titles/chuni/index.py b/titles/chuni/index.py index 080c041..7e72650 100644 --- a/titles/chuni/index.py +++ b/titles/chuni/index.py @@ -150,6 +150,11 @@ class ChuniServlet(BaseServlet): and version_idx >= ChuniConstants.VER_CHUNITHM_NEW ): method_fixed += "C3Exp" + elif ( + isinstance(version, str) + and version.endswith("_chn") + ): + method_fixed += "Chn" hash = PBKDF2( method_fixed, @@ -259,6 +264,13 @@ class ChuniServlet(BaseServlet): internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS elif version >= 135: # LUMINOUS PLUS internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS + elif game_code == "SDHJ": # Chn + if version < 110: # NEW!! + internal_ver = ChuniConstants.VER_CHUNITHM_NEW + elif version >= 110 and version < 120: # NEW PLUS!! + internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS + elif version >= 120: # LUMINOUS + internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32: # If we get a 32 character long hex string, it's a hash and we're @@ -268,6 +280,9 @@ class ChuniServlet(BaseServlet): if game_code == "SDGS": crypto_cfg_key = f"{internal_ver}_int" hash_table_key = f"{internal_ver}_int" + elif game_code == "SDHJ": + crypto_cfg_key = f"{internal_ver}_chn" + hash_table_key = f"{internal_ver}_chn" else: crypto_cfg_key = internal_ver hash_table_key = internal_ver @@ -337,6 +352,8 @@ class ChuniServlet(BaseServlet): endpoint = endpoint.replace("C3Exp", "") elif game_code == "SDGS" and version < 110: endpoint = endpoint.replace("Exp", "") + elif game_code == "SDHJ": + endpoint = endpoint.replace("Chn", "") else: endpoint = endpoint From 4875caab93fac30c4b7a33c9b71b28c94d0b406a Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Sat, 31 May 2025 12:23:40 +1200 Subject: [PATCH 128/130] ChimeDB qr code lookup/userid complete --- core/app.py | 9 +++ core/chimedb.py | 139 +++++++++++++++++++++++++++++++++++++++ core/config.py | 23 +++++++ example_config/core.yaml | 5 ++ 4 files changed, 176 insertions(+) create mode 100644 core/chimedb.py diff --git a/core/app.py b/core/app.py index ecae1bf..4737030 100644 --- a/core/app.py +++ b/core/app.py @@ -11,6 +11,7 @@ from typing import List from core import CoreConfig, TitleServlet, MuchaServlet from core.allnet import AllnetServlet, BillingServlet +from core.chimedb import ChimeServlet from core.frontend import FrontendServlet async def dummy_rt(request: Request): @@ -89,6 +90,14 @@ if not cfg.allnet.standalone: Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini), ] +if cfg.chimedb.enable: + chimedb = ChimeServlet(cfg, cfg_dir) + route_lst += [ + Route("/wc_aime/api/alive_check", chimedb.handle_qr_alive, methods=["POST"]), + Route("/qrcode/api/alive_check", chimedb.handle_qr_alive, methods=["POST"]), + Route("/wc_aime/api/get_data", chimedb.handle_qr_lookup, methods=["POST"]) + ] + for code, game in title.title_registry.items(): route_lst += game.get_routes() diff --git a/core/chimedb.py b/core/chimedb.py new file mode 100644 index 0000000..6e87f69 --- /dev/null +++ b/core/chimedb.py @@ -0,0 +1,139 @@ +import hashlib +import json +import logging +from enum import Enum +from logging.handlers import TimedRotatingFileHandler + +import coloredlogs +from starlette.responses import PlainTextResponse +from starlette.requests import Request + +from core.config import CoreConfig +from core.data import Data + +class ChimeDBStatus(Enum): + NONE = 0 + READER_SETUP_FAIL = 1 + READER_ACCESS_FAIL = 2 + READER_INCOMPATIBLE = 3 + DB_RESOLVE_FAIL = 4 + DB_ACCESS_TIMEOUT = 5 + DB_ACCESS_FAIL = 6 + AIME_ID_INVALID = 7 + NO_BOARD_INFO = 8 + LOCK_BAN_SYSTEM_USER = 9 + LOCK_BAN_SYSTEM = 10 + LOCK_BAN_USER = 11 + LOCK_BAN = 12 + LOCK_SYSTEM_USER = 13 + LOCK_SYSTEM = 14 + LOCK_USER = 15 + +class ChimeServlet: + def __init__(self, core_cfg: CoreConfig, cfg_folder: str) -> None: + self.config = core_cfg + self.config_folder = cfg_folder + + self.data = Data(core_cfg) + + self.logger = logging.getLogger("chimedb") + if not hasattr(self.logger, "initted"): + log_fmt_str = "[%(asctime)s] Chimedb | %(levelname)s | %(message)s" + log_fmt = logging.Formatter(log_fmt_str) + + fileHandler = TimedRotatingFileHandler( + "{0}/{1}.log".format(self.config.server.log_dir, "chimedb"), + when="d", + backupCount=10, + ) + fileHandler.setFormatter(log_fmt) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(log_fmt) + + self.logger.addHandler(fileHandler) + self.logger.addHandler(consoleHandler) + + self.logger.setLevel(self.config.aimedb.loglevel) + coloredlogs.install( + level=core_cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str + ) + self.logger.initted = True + + if not core_cfg.chimedb.key: + self.logger.error("!!!KEY NOT SET!!!") + exit(1) + + self.logger.info("Serving") + + async def handle_qr_alive(self, request: Request): + return PlainTextResponse("alive") + + async def handle_qr_lookup(self, request: Request) -> bytes: + req = json.loads(await request.body()) + access_code = req["qrCode"][-20:] + timestamp = req["timestamp"] + + try: + userId = await self._lookup(access_code) + data = json.dumps({ + "userID": userId, + "errorID": 0, + "timestamp": timestamp, + "key": self._hash_key(userId, timestamp) + }) + except Exception as e: + + self.logger.error(e.with_traceback(None)) + + data = json.dumps({ + "userID": -1, + "errorID": ChimeDBStatus.DB_ACCESS_FAIL, + "timestamp": timestamp, + "key": self._hash_key(-1, timestamp) + }) + + return PlainTextResponse(data) + + def _hash_key(self, chip_id, timestamp): + input_string = f"{chip_id}{timestamp}{self.config.chimedb.key}" + hash_object = hashlib.sha256(input_string.encode('utf-8')) + hex_dig = hash_object.hexdigest() + + formatted_hex = format(int(hex_dig, 16), '064x').upper() + + return formatted_hex + + async def _lookup(self, access_code): + user_id = await self.data.card.get_user_id_from_card(access_code) + + self.logger.info(f"access_code {access_code} -> user_id {user_id}") + + if not user_id or user_id <= 0: + user_id = await self._register(access_code) + + return user_id + + async def _register(self, access_code): + user_id = -1 + + if self.config.server.allow_user_registration: + user_id = await self.data.user.create_user() + + if user_id is None: + self.logger.error("Failed to register user!") + user_id = -1 + else: + card_id = await self.data.card.create_card(user_id, access_code) + + if card_id is None: + self.logger.error("Failed to register card!") + user_id = -1 + + self.logger.info( + f"Register access code {access_code} -> user_id {user_id}" + ) + else: + self.logger.info(f"Registration blocked!: access code {access_code}") + + return user_id diff --git a/core/config.py b/core/config.py index 7b6833d..2b45ed6 100644 --- a/core/config.py +++ b/core/config.py @@ -474,6 +474,28 @@ class AimedbConfig: self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400 ) +class ChimedbConfig: + def __init__(self, parent_config: "CoreConfig") -> None: + self.__config = parent_config + + @property + def enable(self) -> bool: + return CoreConfig.get_config_field( + self.__config, "core", "chimedb", "enable", default=True + ) + @property + def loglevel(self) -> int: + return CoreConfig.str_to_loglevel( + CoreConfig.get_config_field( + self.__config, "core", "chimedb", "loglevel", default="info" + ) + ) + @property + def key(self) -> str: + return CoreConfig.get_config_field( + self.__config, "core", "chimedb", "key", default="" + ) + class MuchaConfig: def __init__(self, parent_config: "CoreConfig") -> None: self.__config = parent_config @@ -495,6 +517,7 @@ class CoreConfig(dict): self.allnet = AllnetConfig(self) self.billing = BillingConfig(self) self.aimedb = AimedbConfig(self) + self.chimedb = ChimedbConfig(self) self.mucha = MuchaConfig(self) @classmethod diff --git a/example_config/core.yaml b/example_config/core.yaml index 53063e3..5042c61 100644 --- a/example_config/core.yaml +++ b/example_config/core.yaml @@ -65,5 +65,10 @@ aimedb: id_secret: "" id_lifetime_seconds: 86400 +chimedb: + enable: False + loglevel: "info" + key: "" + mucha: loglevel: "info" From 33b7db0e985ecc5d04961d137a9de5120b9c6268 Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Sat, 31 May 2025 20:04:28 +1200 Subject: [PATCH 129/130] Added support for multiple Allnet Lite keys + extras --- core/allnet.py | 42 +++++++++++++++++++++++++++----------- core/config.py | 6 +++--- docs/config.md | 12 +++++++++++ docs/game_specific_info.md | 1 + titles/chuni/index.py | 3 +++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 96c5631..7cd9b43 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -139,17 +139,26 @@ class AllnetServlet: useragent_header = request.headers.get('User-Agent', "") is_dfi = pragma_header == "DFI" is_lite = useragent_header[5:] == "Windows/Lite" + lite_id = useragent_header[:4] data = await request.body() - if not self.config.allnet.allnet_lite_key and is_lite: - self.logger.error("!!!LITE KEY NOT SET!!!") - os._exit(1) + if not self.config.allnet.allnet_lite_keys and is_lite: + self.logger.error("!!!LITE KEYS NOT SET!!!") + raise AllnetRequestException() + elif is_lite: + for gameids, key in self.config.allnet.allnet_lite_keys.items(): + if gameids == lite_id: + litekey = key + + if is_lite and "litekey" not in locals(): + self.logger.error("!!!UNIQUE LITE KEY NOT FOUND!!!") + raise AllnetRequestException() try: if is_dfi: req_urlencode = self.from_dfi(data) elif is_lite: - req_urlencode = self.dec_lite(self.config.allnet.allnet_lite_key, data[:16], data) + req_urlencode = self.dec_lite(litekey, data[:16], data) else: req_urlencode = data @@ -348,7 +357,7 @@ class AllnetServlet: ) elif is_lite: iv = bytes([random.randint(2, 255) for _ in range(16)]) - return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp_str)) + return PlainTextResponse(content=self.enc_lite(litekey, iv, resp_str)) return PlainTextResponse(resp_str) @@ -358,17 +367,26 @@ class AllnetServlet: useragent_header = request.headers.get('User-Agent', "") is_dfi = pragma_header == "DFI" is_lite = useragent_header[5:] == "Windows/Lite" + lite_id = useragent_header[:4] data = await request.body() - if not self.config.allnet.allnet_lite_key and is_lite: - self.logger.error("!!!LITE KEY NOT SET!!!") - os._exit(1) + if not self.config.allnet.allnet_lite_keys and is_lite: + self.logger.error("!!!LITE KEYS NOT SET!!!") + raise AllnetRequestException() + elif is_lite: + for gameids, key in self.config.allnet.allnet_lite_keys.items(): + if gameids == lite_id: + litekey = key + + if is_lite and "litekey" not in locals(): + self.logger.error("!!!UNIQUE LITE KEY NOT FOUND!!!") + raise AllnetRequestException() try: if is_dfi: req_urlencode = self.from_dfi(data) elif is_lite: - req_urlencode = self.dec_lite(self.config.allnet.allnet_lite_key, data[:16], data) + req_urlencode = self.dec_lite(litekey, data[:16], data) else: req_urlencode = data.decode() @@ -412,7 +430,7 @@ class AllnetServlet: ) elif is_lite: iv = bytes([random.randint(2, 255) for _ in range(16)]) - return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp)) + return PlainTextResponse(content=self.enc_lite(litekey, iv, resp)) return PlainTextResponse(resp) else: @@ -425,7 +443,7 @@ class AllnetServlet: ) elif is_lite: iv = bytes([random.randint(2, 255) for _ in range(16)]) - return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, resp)) + return PlainTextResponse(content=self.enc_lite(litekey, iv, resp)) return PlainTextResponse(resp) if path.exists( @@ -457,7 +475,7 @@ class AllnetServlet: ) elif is_lite: iv = bytes([random.randint(2, 255) for _ in range(16)]) - return PlainTextResponse(content=self.enc_lite(self.config.allnet.allnet_lite_key, iv, res_str)) + return PlainTextResponse(content=self.enc_lite(litekey, iv, res_str)) return PlainTextResponse(res_str) diff --git a/core/config.py b/core/config.py index 2b45ed6..f79d6c0 100644 --- a/core/config.py +++ b/core/config.py @@ -1,7 +1,7 @@ import logging import os import ssl -from typing import Any, Union +from typing import Any, Union, Dict from typing_extensions import Optional @@ -379,9 +379,9 @@ class AllnetConfig: self.__config, "core", "allnet", "save_billing", default=False ) @property - def allnet_lite_key(self) -> bool: + def allnet_lite_keys(self) -> Dict: return CoreConfig.get_config_field( - self.__config, "core", "allnet", "allnet_lite_key", default=[] + self.__config, "core", "allnet", "allnet_lite_keys", default={} ) class BillingConfig: diff --git a/docs/config.md b/docs/config.md index f85e8e7..6cf2482 100644 --- a/docs/config.md +++ b/docs/config.md @@ -41,6 +41,13 @@ - `loglevel`: Logging level for the allnet server. Default `info` - `allow_online_updates`: Allow allnet to distribute online updates via DownloadOrders. This system is currently non-functional, so leave it disabled. Default `False` - `update_cfg_folder`: Folder where delivery INI files will be checked for. Ignored if `allow_online_updates` is `False`. Default `""` +- `allnet_lite_keys:` Allnet Lite (Chinese Allnet) PowerOn/DownloadOrder unique keys. Default ` ` +```yaml + allnet_lite_keys: + "SDJJ": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] + "SDHJ": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] + "SDGB": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] +``` ## Billing - `standalone`: Whether the billing server should launch it's own servlet on it's own port, or be part of the main servlet on the default port. Setting this to `True` requires that you have `ssl_key` and `ssl_cert` set. Default `False` - `loglevel`: Logging level for the billing server. Default `info` @@ -56,3 +63,8 @@ - `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""` - `id_secret`: Base64-encoded JWT secret for Sega Auth IDs. Leaving this blank disables this feature. Default `""` - `id_lifetime_seconds`: Number of secons a JWT generated should be valid for. Default `86400` (1 day) +## Chimedb +- `enable`: Whether or not chimedb should run. Default `False` +- `loglevel`: Logging level for the chimedb server. Default `info` +- `key`: Key to hash chimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle chimedb requests. Default `""` + diff --git a/docs/game_specific_info.md b/docs/game_specific_info.md index 7121478..0903979 100644 --- a/docs/game_specific_info.md +++ b/docs/game_specific_info.md @@ -108,6 +108,7 @@ crypto: keys: 13: ["0000000000000000000000000000000000000000000000000000000000000000", "00000000000000000000000000000000", "0000000000000000"] "13_int": ["0000000000000000000000000000000000000000000000000000000000000000", "00000000000000000000000000000000", "0000000000000000", 42] + "13_chn": ["0000000000000000000000000000000000000000000000000000000000000000", "00000000000000000000000000000000", "0000000000000000", 8] ``` ### Database upgrade diff --git a/titles/chuni/index.py b/titles/chuni/index.py index 7e72650..ad51bb0 100644 --- a/titles/chuni/index.py +++ b/titles/chuni/index.py @@ -101,14 +101,17 @@ class ChuniServlet(BaseServlet): f"{ChuniConstants.VER_CHUNITHM_PARADISE}_int": 51, # SUPERSTAR PLUS ChuniConstants.VER_CHUNITHM_NEW: 54, f"{ChuniConstants.VER_CHUNITHM_NEW}_int": 49, + f"{ChuniConstants.VER_CHUNITHM_NEW}_chn": 37, ChuniConstants.VER_CHUNITHM_NEW_PLUS: 25, f"{ChuniConstants.VER_CHUNITHM_NEW_PLUS}_int": 31, + f"{ChuniConstants.VER_CHUNITHM_NEW_PLUS}_chn": 35, ChuniConstants.VER_CHUNITHM_SUN: 70, f"{ChuniConstants.VER_CHUNITHM_SUN}_int": 35, ChuniConstants.VER_CHUNITHM_SUN_PLUS: 36, f"{ChuniConstants.VER_CHUNITHM_SUN_PLUS}_int": 36, ChuniConstants.VER_CHUNITHM_LUMINOUS: 8, f"{ChuniConstants.VER_CHUNITHM_LUMINOUS}_int": 8, + f"{ChuniConstants.VER_CHUNITHM_LUMINOUS}_chn": 8, ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS: 56, } From 02bfc7dba250e4d85bc7d0bbbce6aaa8b068c1b7 Mon Sep 17 00:00:00 2001 From: Keeboy99 <67620144+Keeboy99@users.noreply.github.com> Date: Mon, 2 Jun 2025 16:31:57 +1200 Subject: [PATCH 130/130] SDGB support + extras --- core/allnet.py | 4 ++-- readme.md | 29 +++++++++++++++++++++++++++++ titles/chuni/index.py | 6 +++--- titles/mai2/__init__.py | 1 + titles/mai2/base.py | 3 +++ titles/mai2/const.py | 1 + titles/mai2/index.py | 16 ++++++++++++++++ 7 files changed, 55 insertions(+), 5 deletions(-) diff --git a/core/allnet.py b/core/allnet.py index 7cd9b43..94b34ab 100644 --- a/core/allnet.py +++ b/core/allnet.py @@ -606,9 +606,9 @@ class AllnetServlet: return decrypted[16:].decode("utf-8") def enc_lite(self, key, iv, data): - decrypted = pad(bytes([0] * 16) + data.encode('utf-8'), 16) + unencrypted = pad(bytes([0] * 16) + data.encode('utf-8'), 16) cipher = AES.new(bytes(key), AES.MODE_CBC, iv) - encrypted = cipher.encrypt(decrypted) + encrypted = cipher.encrypt(unencrypted) return encrypted class BillingServlet: diff --git a/readme.md b/readme.md index e29784d..8591c74 100644 --- a/readme.md +++ b/readme.md @@ -8,6 +8,11 @@ Games listed below have been tested and confirmed working. Only game versions ol + 1.30 + 1.35 ++ CHUNITHM CHINA + + NEW + + 2024 (NEW) + + 2024 (LUMINOUS) + + CHUNITHM INTL + SUPERSTAR + SUPERSTAR PLUS @@ -15,6 +20,8 @@ Games listed below have been tested and confirmed working. Only game versions ol + NEW PLUS + SUN + SUN PLUS + + LUMINOUS + + LUMINOUS PLUS + CHUNITHM JP + AIR @@ -43,7 +50,29 @@ Games listed below have been tested and confirmed working. Only game versions ol + Initial D THE ARCADE + Season 2 ++ maimai DX CHINA + + DX (Muji) + + 2021 (Muji) + + 2022 (Muji) + + 2023 (FESTiVAL) + + 2024 (BUDDiES) + ++ maimai DX INTL + + DX + + DX Plus + + Splash + + Splash Plus + + UNiVERSE + + UNiVERSE PLUS + + FESTiVAL + + FESTiVAL PLUS + + BUDDiES + + BUDDiES PLUS + + PRiSM + + maimai DX + + DX + + DX Plus + Splash + Splash Plus + UNiVERSE diff --git a/titles/chuni/index.py b/titles/chuni/index.py index ad51bb0..1392588 100644 --- a/titles/chuni/index.py +++ b/titles/chuni/index.py @@ -104,7 +104,7 @@ class ChuniServlet(BaseServlet): f"{ChuniConstants.VER_CHUNITHM_NEW}_chn": 37, ChuniConstants.VER_CHUNITHM_NEW_PLUS: 25, f"{ChuniConstants.VER_CHUNITHM_NEW_PLUS}_int": 31, - f"{ChuniConstants.VER_CHUNITHM_NEW_PLUS}_chn": 35, + f"{ChuniConstants.VER_CHUNITHM_NEW_PLUS}_chn": 35, # NEW ChuniConstants.VER_CHUNITHM_SUN: 70, f"{ChuniConstants.VER_CHUNITHM_SUN}_int": 35, ChuniConstants.VER_CHUNITHM_SUN_PLUS: 36, @@ -268,9 +268,9 @@ class ChuniServlet(BaseServlet): elif version >= 135: # LUMINOUS PLUS internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS_PLUS elif game_code == "SDHJ": # Chn - if version < 110: # NEW!! + if version < 110: # NEW internal_ver = ChuniConstants.VER_CHUNITHM_NEW - elif version >= 110 and version < 120: # NEW PLUS!! + elif version >= 110 and version < 120: # NEW *Cursed but needed due to different encryption key internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS elif version >= 120: # LUMINOUS internal_ver = ChuniConstants.VER_CHUNITHM_LUMINOUS diff --git a/titles/mai2/__init__.py b/titles/mai2/__init__.py index 234e864..4c1739d 100644 --- a/titles/mai2/__init__.py +++ b/titles/mai2/__init__.py @@ -18,4 +18,5 @@ game_codes = [ Mai2Constants.GAME_CODE_GREEN, Mai2Constants.GAME_CODE, Mai2Constants.GAME_CODE_DX_INT, + Mai2Constants.GAME_CODE_DX_CHN, ] diff --git a/titles/mai2/base.py b/titles/mai2/base.py index 5d1c767..1983ef7 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -139,6 +139,9 @@ class Mai2Base: async def handle_get_game_ng_music_id_api_request(self, data: Dict) -> Dict: return {"length": 0, "musicIdList": []} + async def handle_get_game_ng_word_list_api_request(self, data: Dict) -> Dict: + return {"ngWordExactMatchLength": 0, "ngWordExactMatchList": [], "ngWordPartialMatchLength": 0, "ngWordPartialMatchList": []} + async def handle_get_game_charge_api_request(self, data: Dict) -> Dict: game_charge_list = await self.data.static.get_enabled_tickets(self.version, 1) if game_charge_list is None: diff --git a/titles/mai2/const.py b/titles/mai2/const.py index 99642b2..47e5cbd 100644 --- a/titles/mai2/const.py +++ b/titles/mai2/const.py @@ -32,6 +32,7 @@ class Mai2Constants: GAME_CODE_FINALE = "SDEY" GAME_CODE_DX = "SDEZ" GAME_CODE_DX_INT = "SDGA" + GAME_CODE_DX_CHN = "SDGB" CONFIG_NAME = "mai2.yaml" diff --git a/titles/mai2/index.py b/titles/mai2/index.py index d8e2a4f..d59ce87 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -337,6 +337,20 @@ class Mai2Servlet(BaseServlet): elif version >=150: internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM + elif game_code == "SDGB": # Chn + if version < 110: # Muji + internal_ver = Mai2Constants.VER_MAIMAI_DX + elif version >= 110 and version < 120: # Muji + internal_ver = Mai2Constants.VER_MAIMAI_DX + elif version >= 120 and version < 130: # Muji (LMAO) + internal_ver = Mai2Constants.VER_MAIMAI_DX + elif version >= 130 and version < 140: # FESTiVAL + internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL + elif version >= 140 and version < 150: # BUDDiES + internal_ver = Mai2Constants.VER_MAIMAI_DX_BUDDIES + elif version >=150: + internal_ver = Mai2Constants.VER_MAIMAI_DX_PRISM + if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32: # If we get a 32 character long hex string, it's a hash and we're # dealing with an encrypted request. False positives shouldn't happen @@ -403,6 +417,8 @@ class Mai2Servlet(BaseServlet): endpoint = ( endpoint.replace("MaimaiExp", "") if game_code == Mai2Constants.GAME_CODE_DX_INT + else endpoint.replace("MaimaiChn", "") + if game_code == Mai2Constants.GAME_CODE_DX_CHN else endpoint ) func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"