From 36944984e89954cfb23a304e274666707ceb314a Mon Sep 17 00:00:00 2001 From: drmext <71258889+drmext@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:38:12 +0000 Subject: [PATCH] Fix --- config.py | 2 +- core_database.py | 34 ++++++++++++++++- modules/ddr/api.py | 35 +++++++++++------ modules/gitadora/api.py | 11 +++--- modules/iidx/api.py | 21 +++++++---- modules/webapi.py | 16 ++++++-- pyeamu.py | 84 ++++++++++++++++++++++------------------- requirements.txt | 2 - 8 files changed, 133 insertions(+), 72 deletions(-) diff --git a/config.py b/config.py index d63be04..010a8d5 100644 --- a/config.py +++ b/config.py @@ -16,7 +16,7 @@ def get_ip(): ip = get_ip() port = 8000 -response_compression = True +response_compression = False verbose_log = False dev_reload = False diff --git a/core_database.py b/core_database.py index b961a1f..da55328 100644 --- a/core_database.py +++ b/core_database.py @@ -1,7 +1,37 @@ -from tinydb import TinyDB +"""TinyDB kept in memory. -db = TinyDB("db.json", indent=2, encoding="utf-8", ensure_ascii=False) +Each TinyDB operation used to reread and rewrite db.json. The cache serves +reads from memory and holds writes until flush_db(), which runs once per +request and again on process shutdown. +""" + +import atexit + +from tinydb import TinyDB +from tinydb.middlewares import CachingMiddleware +from tinydb.storages import JSONStorage + + +class _CachedJSONStorage(CachingMiddleware): + # Do not flush after N writes. flush_db() is the only flush. + WRITE_CACHE_SIZE = 2**31 + + +db = TinyDB( + "db.json", + indent=2, + encoding="utf-8", + ensure_ascii=False, + storage=_CachedJSONStorage(JSONStorage), +) def get_db(): return db + + +def flush_db() -> None: + db.storage.flush() + + +atexit.register(flush_db) diff --git a/modules/ddr/api.py b/modules/ddr/api.py index cabc9b9..6c4df42 100644 --- a/modules/ddr/api.py +++ b/modules/ddr/api.py @@ -1,10 +1,10 @@ +from dataclasses import dataclass from typing import Optional, Dict, List, Tuple from os import path import json import struct import lxml.etree as ET -from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route, Router @@ -12,16 +12,18 @@ from tinydb import where import utils.card as conv from core_database import get_db -from modules.webapi import endpoint, read_model +from modules.webapi import MAX_UPLOAD_BYTES, endpoint, read_model from utils.lz77 import lz77_decode -class DDR_Profile_Main_Items(BaseModel): +@dataclass +class DDR_Profile_Main_Items: card: str pin: str -class DDR_Profile_19_Items(BaseModel): +@dataclass +class DDR_Profile_19_Items: game_version: Optional[int] calories_disp: Optional[bool] character: Optional[str] @@ -41,7 +43,8 @@ class DDR_Profile_19_Items(BaseModel): double_grade: Optional[int] -class DDR_Profile_20_Items(BaseModel): +@dataclass +class DDR_Profile_20_Items: game_version: Optional[int] common_dancername: Optional[str] common_area: Optional[str] @@ -51,6 +54,13 @@ class DDR_Profile_20_Items(BaseModel): customize: Optional[dict] +def _on_off(value) -> str: + """Accept bool or On/Off-like strings from the webui; store canonical On/Off.""" + if isinstance(value, str): + return "On" if value.strip().lower() in ("on", "true", "1") else "Off" + return "On" if value else "Off" + + @endpoint async def ddr_profiles(request: Request): return get_db().table("ddr_profile").all() @@ -86,13 +96,13 @@ async def ddr_profile_id_19_patch(request: Request): game_profile = profile["version"].get("19", {}) game_profile["game_version"] = item.game_version - game_profile["calories_disp"] = "On" if item.calories_disp else "Off" + game_profile["calories_disp"] = _on_off(item.calories_disp) game_profile["character"] = item.character game_profile["arrow_skin"] = item.arrow_skin game_profile["filter"] = item.filter game_profile["guideline"] = item.guideline game_profile["priority"] = item.priority - game_profile["timing_disp"] = "On" if item.timing_disp else "Off" + game_profile["timing_disp"] = _on_off(item.timing_disp) game_profile["common"] = item.common game_profile["option"] = item.option game_profile["last"] = item.last @@ -241,9 +251,7 @@ class ARC: @endpoint async def ddr_receive_mdb(request: Request): - form = await request.form() - upload = form["file"] - data = await upload.read() + data = await request.body() arc = ARC(data) try: mdb_new = ET.fromstring( @@ -312,6 +320,11 @@ router = Router( Route("/scores_best/{ddr_id}", ddr_scores_best_id, methods=["GET"]), Route("/mcode/{mcode}/all", ddr_scores_mcode_all, methods=["GET"]), Route("/mcode/{mcode}/best", ddr_scores_mcode_best, methods=["GET"]), - Route("/parse_mdb/upload", ddr_receive_mdb, methods=["POST"]), + Route( + "/parse_mdb/upload", + ddr_receive_mdb, + methods=["POST"], + max_body_size=MAX_UPLOAD_BYTES, + ), ] ) diff --git a/modules/gitadora/api.py b/modules/gitadora/api.py index 2294595..567b74d 100644 --- a/modules/gitadora/api.py +++ b/modules/gitadora/api.py @@ -1,6 +1,5 @@ -from typing import Optional +from dataclasses import dataclass, field -from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route, Router @@ -11,16 +10,18 @@ from core_database import get_db from modules.webapi import endpoint, read_model -class GFDM_Profile_Main_Items(BaseModel): +@dataclass +class GFDM_Profile_Main_Items: card: str pin: str -class GFDM_Profile_Version_Items(BaseModel): +@dataclass +class GFDM_Profile_Version_Items: game_version: int name: str title: str - rival_card_ids: list = [] + rival_card_ids: list = field(default_factory=list) @endpoint diff --git a/modules/iidx/api.py b/modules/iidx/api.py index cdd4772..b487fbf 100644 --- a/modules/iidx/api.py +++ b/modules/iidx/api.py @@ -1,9 +1,9 @@ +from dataclasses import dataclass from typing import Optional from os import path import json import xml.etree.ElementTree as ET -from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route, Router @@ -12,15 +12,17 @@ from tinydb import where import utils.card as conv import utils.musicdata_tool as mdt from core_database import get_db -from modules.webapi import endpoint, read_model +from modules.webapi import MAX_UPLOAD_BYTES, endpoint, read_model -class IIDX_Profile_Main_Items(BaseModel): +@dataclass +class IIDX_Profile_Main_Items: card: str pin: str -class IIDX_Profile_Version_Items(BaseModel): +@dataclass +class IIDX_Profile_Version_Items: djname: Optional[str] region: Optional[int] head: Optional[int] @@ -256,9 +258,7 @@ async def iidx_score_stats_song(request: Request): @endpoint async def iidx_receive_mdb(request: Request): - form = await request.form() - upload = form["file"] - data = await upload.read() + data = await request.body() iidx_bin = path.join("webui", "music_data.bin") iidx_vid = path.join("webui", "video_music_list.xml") @@ -346,6 +346,11 @@ router = Router( Route("/class_best/{iidx_id}", iidx_class_best, methods=["GET"]), Route("/score_stats/all", iidx_score_stats, methods=["GET"]), Route("/score_stats/{music_id}", iidx_score_stats_song, methods=["GET"]), - Route("/parse_mdb/upload", iidx_receive_mdb, methods=["POST"]), + Route( + "/parse_mdb/upload", + iidx_receive_mdb, + methods=["POST"], + max_body_size=MAX_UPLOAD_BYTES, + ), ] ) diff --git a/modules/webapi.py b/modules/webapi.py index cc0e354..26104e5 100644 --- a/modules/webapi.py +++ b/modules/webapi.py @@ -3,17 +3,25 @@ from __future__ import annotations import json -from typing import Any, Callable, Type +from dataclasses import fields +from typing import Any, Callable, Type, TypeVar -from pydantic import BaseModel from starlette.requests import Request from starlette.responses import JSONResponse, Response +# Music-data uploads are held in memory, so reject anything larger than this. +MAX_UPLOAD_BYTES = 128 * 1024 * 1024 -async def read_model(request: Request, model: Type[BaseModel]) -> BaseModel: +T = TypeVar("T") + + +async def read_model(request: Request, model: Type[T]) -> T: body = await request.body() data = json.loads(body) if body else {} - return model.model_validate(data) + if not isinstance(data, dict): + raise ValueError("JSON body must be an object") + names = {field.name for field in fields(model)} + return model(**{key: value for key, value in data.items() if key in names}) def as_response(value: Any) -> Response: diff --git a/pyeamu.py b/pyeamu.py index c1d03ba..9ecf5b3 100644 --- a/pyeamu.py +++ b/pyeamu.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import re from os import path from urllib.parse import unquote, urlencode, urlparse, urlunparse @@ -19,6 +18,7 @@ import config import modules import utils.card as conv from core_common import E, EamuseError, core_prepare_response, core_process_request +from core_database import flush_db from modules.registry import dispatch, iter_services loopback = "127.0.0.1" @@ -186,8 +186,6 @@ def _build_webui() -> Starlette: routes.insert(0, Mount(mount_path, app=api_router)) if path.exists("webui"): - with open(path.join("webui", "monkey.json"), "w") as f: - json.dump(settings, f, indent=2) routes.insert(0, Mount("/webui", app=StaticFiles(directory="webui", html=True))) else: routes.insert(0, Route("/webui", redirect_webui_missing, methods=["GET"])) @@ -214,46 +212,54 @@ async def _send_response(response: Response, scope: Scope, receive: Receive, sen async def app(scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] == "lifespan": - await webui_app(scope, receive, send) + + async def send_with_flush(message): + if message["type"] == "lifespan.shutdown.complete": + flush_db() + await send(message) + + await webui_app(scope, receive, send_with_flush) return if scope["type"] != "http": return - raw_path = unquote(scope.get("path") or "") - method = scope.get("method", "GET").upper() - - # WebUI / static / config / card converter - if raw_path == "/" or any( - raw_path == p or raw_path.startswith(p + "/") for p in _WEBUI_PREFIXES - ): - await webui_app(scope, receive, send) - return - - if method != "POST": - await _send_response(Response(status_code=404), scope, receive, send) - return - - request = Request(scope, receive) - + response: Response | None = None try: - # services.get - if raw_path.rstrip("/") == "/core" or _CORE_SERVICES_GET.match(raw_path): - response = await services_get(request) - elif raw_path.rstrip("/") == "/fwdr": - response = await handle_xrpc(request) - else: - # slashed: /{svc}/{gameinfo}/{path...} — model is the gameinfo segment - parts = [p for p in raw_path.strip("/").split("/") if p] - if len(parts) >= 3: - scope = dict(scope) - scope["path_params"] = {"path": "/".join(parts[2:])} - request = Request(scope, receive) - model = request.query_params.get("model") or parts[1] - response = await handle_xrpc(request, model=model) - else: - response = Response(status_code=404) - except EamuseError as exc: - response = Response(status_code=exc.status_code, content=exc.detail or b"") + raw_path = unquote(scope.get("path") or "") + method = scope.get("method", "GET").upper() - await _send_response(response, scope, receive, send) + # WebUI / static / config / card converter + if raw_path == "/" or any( + raw_path == p or raw_path.startswith(p + "/") for p in _WEBUI_PREFIXES + ): + await webui_app(scope, receive, send) + elif method != "POST": + response = Response(status_code=404) + else: + request = Request(scope, receive) + + try: + # services.get + if raw_path.rstrip("/") == "/core" or _CORE_SERVICES_GET.match(raw_path): + response = await services_get(request) + elif raw_path.rstrip("/") == "/fwdr": + response = await handle_xrpc(request) + else: + # slashed: /{svc}/{gameinfo}/{path...} — model is the gameinfo segment + parts = [p for p in raw_path.strip("/").split("/") if p] + if len(parts) >= 3: + scope = dict(scope) + scope["path_params"] = {"path": "/".join(parts[2:])} + request = Request(scope, receive) + model = request.query_params.get("model") or parts[1] + response = await handle_xrpc(request, model=model) + else: + response = Response(status_code=404) + except EamuseError as exc: + response = Response(status_code=exc.status_code, content=exc.detail or b"") + finally: + flush_db() + + if response is not None: + await _send_response(response, scope, receive, send) diff --git a/requirements.txt b/requirements.txt index 07d7df4..794971f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,4 @@ starlette uvicorn[standard] kbinxml>=2.0 pycryptodomex -python-multipart -pydantic tinydb