Initial commit of BEMANI Utilities to GitHub.

bemani/data/api/base.py

@@ -0,0 +1,19 @@
+from typing import List, Optional
+
+from bemani.data.api.client import APIClient
+from bemani.data.interfaces import APIProviderInterface
+
+
+class BaseGlobalData:
+
+    def __init__(self, api: APIProviderInterface) -> None:
+        self.__localapi = api
+        self.__apiclients: Optional[List[APIClient]] = None
+
+    @property
+    def clients(self) -> List[APIClient]:
+        if self.__apiclients is None:
+            servers = self.__localapi.get_all_servers()
+            self.__apiclients = [APIClient(server.uri, server.token, server.allow_stats, server.allow_scores) for server in servers]
+
+        return self.__apiclients

bemani/data/api/client.py

@@ -0,0 +1,268 @@
+import json
+import requests
+from typing import Tuple, Dict, List, Any, Optional
+
+from bemani.common import GameConstants, VersionConstants, DBConstants, ValidatedDict
+
+
+class APIException(Exception):
+    pass
+
+
+class NotAuthorizedAPIException(APIException):
+    pass
+
+
+class UnsupportedRequestAPIException(APIException):
+    pass
+
+
+class UnrecognizedRequestAPIException(APIException):
+    pass
+
+
+class UnsupportedVersionAPIException(APIException):
+    pass
+
+
+class RemoteServerErrorAPIException(APIException):
+    pass
+
+
+class APIClient:
+    """
+    A client that fully speaks BEMAPI and can pull information from a remote server.
+    """
+
+    API_VERSION = 'v1'
+
+    def __init__(self, base_uri: str, token: str, allow_stats: bool, allow_scores: bool) -> None:
+        self.base_uri = base_uri
+        self.token = token
+        self.allow_stats = allow_stats
+        self.allow_scores = allow_scores
+
+    def __exchange_data(self, request_uri: str, request_args: Dict[str, Any]) -> Dict[str, Any]:
+        if self.base_uri[-1:] != '/':
+            uri = '{}/{}'.format(self.base_uri, request_uri)
+        else:
+            uri = '{}{}'.format(self.base_uri, request_uri)
+
+        headers = {
+            'Authorization': 'Token {}'.format(self.token),
+            'Content-Type': 'application/json; charset=utf-8',
+        }
+        data = json.dumps(request_args).encode('utf8')
+
+        try:
+            r = requests.request(
+                'GET',
+                uri,
+                headers=headers,
+                data=data,
+                allow_redirects=False,
+                timeout=10,
+            )
+        except Exception:
+            raise APIException('Failed to query remote server!')
+
+        if r.headers['content-type'] != 'application/json; charset=utf-8':
+            raise APIException('API returned invalid content type \'{}\'!'.format(r.headers['content-type']))
+
+        jsondata = r.json()
+
+        if r.status_code == 200:
+            return jsondata
+
+        if 'error' not in jsondata:
+            raise APIException('API returned error code {} but did not include \'error\' attribute in response JSON!'.format(r.status_code))
+        error = jsondata['error']
+
+        if r.status_code == 401:
+            raise NotAuthorizedAPIException('The API token used is not authorized against this server!')
+        if r.status_code == 404:
+            raise UnsupportedRequestAPIException('The server does not support this game/version or request object!')
+        if r.status_code == 405:
+            raise UnrecognizedRequestAPIException('The server did not recognize the request!')
+        if r.status_code == 500:
+            raise RemoteServerErrorAPIException('The server had an error processing the request and returned \'{}\''.format(error))
+        if r.status_code == 501:
+            raise UnsupportedVersionAPIException('The server does not support this version of the API!')
+        raise APIException('The server returned an invalid status code {}!', format(r.status_code))
+
+    def __translate(self, game: str, version: int) -> Tuple[str, str]:
+        servergame = {
+            GameConstants.DDR: 'ddr',
+            GameConstants.IIDX: 'iidx',
+            GameConstants.JUBEAT: 'jubeat',
+            GameConstants.MUSECA: 'museca',
+            GameConstants.POPN_MUSIC: 'popnmusic',
+            GameConstants.REFLEC_BEAT: 'reflecbeat',
+            GameConstants.SDVX: 'soundvoltex',
+        }.get(game)
+        if servergame is None:
+            raise UnsupportedRequestAPIException('The client does not support this game/version!')
+
+        if version >= DBConstants.OMNIMIX_VERSION_BUMP:
+            version = version - DBConstants.OMNIMIX_VERSION_BUMP
+            omnimix = True
+        else:
+            omnimix = False
+
+        serverversion = {
+            GameConstants.DDR: {
+                VersionConstants.DDR_X2: '12',
+                VersionConstants.DDR_X3_VS_2NDMIX: '13',
+                VersionConstants.DDR_2013: '14',
+                VersionConstants.DDR_2014: '15',
+                VersionConstants.DDR_ACE: '16',
+            },
+            GameConstants.IIDX: {
+                VersionConstants.IIDX_TRICORO: '20',
+                VersionConstants.IIDX_SPADA: '21',
+                VersionConstants.IIDX_PENDUAL: '22',
+                VersionConstants.IIDX_COPULA: '23',
+                VersionConstants.IIDX_SINOBUZ: '24',
+                VersionConstants.IIDX_CANNON_BALLERS: '25',
+            },
+            GameConstants.JUBEAT: {
+                VersionConstants.JUBEAT_SAUCER: '5',
+                VersionConstants.JUBEAT_SAUCER_FULFILL: '5a',
+                VersionConstants.JUBEAT_PROP: '6',
+                VersionConstants.JUBEAT_QUBELL: '7',
+                VersionConstants.JUBEAT_CLAN: '8',
+            },
+            GameConstants.MUSECA: {
+                VersionConstants.MUSECA: '1',
+                VersionConstants.MUSECA_1_PLUS: '1p',
+            },
+            GameConstants.POPN_MUSIC: {
+                VersionConstants.POPN_MUSIC_TUNE_STREET: '19',
+                VersionConstants.POPN_MUSIC_FANTASIA: '20',
+                VersionConstants.POPN_MUSIC_SUNNY_PARK: '21',
+                VersionConstants.POPN_MUSIC_LAPISTORIA: '22',
+                VersionConstants.POPN_MUSIC_ECLALE: '23',
+                VersionConstants.POPN_MUSIC_USANEKO: '24',
+            },
+            GameConstants.REFLEC_BEAT: {
+                VersionConstants.REFLEC_BEAT: '1',
+                VersionConstants.REFLEC_BEAT_LIMELIGHT: '2',
+                VersionConstants.REFLEC_BEAT_COLETTE: '3as',
+                VersionConstants.REFLEC_BEAT_GROOVIN: '4u',
+                VersionConstants.REFLEC_BEAT_VOLZZA: '5',
+                VersionConstants.REFLEC_BEAT_VOLZZA_2: '5a',
+                VersionConstants.REFLEC_BEAT_REFLESIA: '6',
+            },
+            GameConstants.SDVX: {
+                VersionConstants.SDVX_BOOTH: '1',
+                VersionConstants.SDVX_INFINITE_INFECTION: '2',
+                VersionConstants.SDVX_GRAVITY_WARS: '3',
+                VersionConstants.SDVX_HEAVENLY_HAVEN: '4',
+            },
+        }.get(game, {}).get(version)
+        if serverversion is None:
+            raise UnsupportedRequestAPIException('The client does not support this game/version!')
+
+        if omnimix:
+            serverversion = 'o' + serverversion
+
+        return (servergame, serverversion)
+
+    def get_server_info(self) -> ValidatedDict:
+        resp = self.__exchange_data('', {})
+        return ValidatedDict({
+            'name': resp['name'],
+            'email': resp['email'],
+            'versions': resp['versions'],
+        })
+
+    def get_profiles(self, game: str, version: int, idtype: str, ids: List[str]) -> List[Dict[str, Any]]:
+        # Allow remote servers to be disabled
+        if not self.allow_scores:
+            return []
+
+        try:
+            servergame, serverversion = self.__translate(game, version)
+            resp = self.__exchange_data(
+                '{}/{}/{}'.format(self.API_VERSION, servergame, serverversion),
+                {
+                    'ids': ids,
+                    'type': idtype,
+                    'objects': ['profile'],
+                },
+            )
+            return resp['profile']
+        except APIException:
+            # Couldn't talk to server, assume empty profiles
+            return []
+
+    def get_records(
+        self,
+        game: str,
+        version: int,
+        idtype: str,
+        ids: List[str],
+        since: Optional[int]=None,
+        until: Optional[int]=None,
+    ) -> List[Dict[str, Any]]:
+        # Allow remote servers to be disabled
+        if not self.allow_scores:
+            return []
+
+        try:
+            servergame, serverversion = self.__translate(game, version)
+            data: Dict[str, Any] = {
+                'ids': ids,
+                'type': idtype,
+                'objects': ['records'],
+            }
+            if since is not None:
+                data['since'] = since
+            if until is not None:
+                data['until'] = until
+            resp = self.__exchange_data(
+                '{}/{}/{}'.format(self.API_VERSION, servergame, serverversion),
+                data,
+            )
+            return resp['records']
+        except APIException:
+            # Couldn't talk to server, assume empty records
+            return []
+
+    def get_statistics(self, game: str, version: int, idtype: str, ids: List[str]) -> List[Dict[str, Any]]:
+        # Allow remote servers to be disabled
+        if not self.allow_stats:
+            return []
+
+        try:
+            servergame, serverversion = self.__translate(game, version)
+            resp = self.__exchange_data(
+                '{}/{}/{}'.format(self.API_VERSION, servergame, serverversion),
+                {
+                    'ids': ids,
+                    'type': idtype,
+                    'objects': ['statistics'],
+                },
+            )
+            return resp['statistics']
+        except APIException:
+            # Couldn't talk to server, assume empty statistics
+            return []
+
+    def get_catalog(self, game: str, version: int) -> Dict[str, List[Dict[str, Any]]]:
+        # No point disallowing this, since its only ever used for bootstrapping.
+
+        try:
+            servergame, serverversion = self.__translate(game, version)
+            resp = self.__exchange_data(
+                '{}/{}/{}'.format(self.API_VERSION, servergame, serverversion),
+                {
+                    'ids': [],
+                    'type': 'server',
+                    'objects': ['catalog'],
+                },
+            )
+            return resp['catalog']
+        except APIException:
+            # Couldn't talk to server, assume empty catalog
+            return {}

bemani/data/api/game.py

@@ -0,0 +1,94 @@
+from typing import List, Optional, Dict, Any, Set
+
+from bemani.common import GameConstants, ValidatedDict, Parallel
+from bemani.data.api.base import BaseGlobalData
+from bemani.data.types import Item
+
+
+class GlobalGameData(BaseGlobalData):
+
+    def __translate_sdvx_song_unlock(self, entry: Dict[str, Any]) -> Item:
+        return Item(
+            "song_unlock",
+            int(entry["catalogid"]),
+            {
+                "musicid": int(entry["song"]),
+                "chart": int(entry["chart"]),
+                "blocks": int(entry["price"]),
+            },
+        )
+
+    def __translate_sdvx_appealcard(self, entry: Dict[str, Any]) -> Item:
+        return Item(
+            "appealcard",
+            int(entry["appealid"]),
+            {},
+        )
+
+    def get_items(self, game: str, version: int) -> List[Item]:
+        """
+        Given a game/userid, find all items in the catalog.
+
+        Parameters:
+            game - String identifier of the game looking up the catalog.
+            version - Integer identifier of the version looking up this catalog.
+
+        Returns:
+            A list of item objects.
+        """
+        catalogs: List[Dict[str, List[Dict[str, Any]]]] = Parallel.call(
+            [client.get_catalog for client in self.clients],
+            game,
+            version
+        )
+        retval: List[Item] = []
+        seen: Set[str] = set()
+        for catalog in catalogs:
+            for catalogtype in catalog:
+                # Simple LUT for now, might need to be complicated later
+                if game == GameConstants.SDVX:
+                    translation = {
+                        "purchases": self.__translate_sdvx_song_unlock,
+                        "appealcards": self.__translate_sdvx_appealcard,
+                    }.get(catalogtype, None)
+                else:
+                    translation = None
+
+                # If we don't have a mapping for this, ignore it
+                if translation is None:
+                    continue
+
+                for entry in catalog[catalogtype]:
+                    # Translate the entry
+                    item = translation(entry)
+
+                    # Now, see if it is unique, and if so, remember it
+                    key = f"{item.type}_{item.id}"
+                    if key in seen:
+                        continue
+
+                    retval.append(item)
+                    seen.add(key)
+        return retval
+
+    def get_item(self, game: str, version: int, catid: int, cattype: str) -> Optional[ValidatedDict]:
+        """
+        Given a game/userid and catalog id/type, find that catalog entry.
+
+        Note that there can be more than one catalog entry with the same ID and game/userid
+        as long as each one is a different type. Essentially, cattype namespaces catalog entry.
+
+        Parameters:
+            game - String identifier of the game looking up this entry.
+            version - Integer identifier of the version looking up this entry.
+            catid - Integer ID, as provided by a game.
+            cattype - The type of catalog entry.
+
+        Returns:
+            A dictionary as stored by a game class previously, or None if not found.
+        """
+        all_items = self.get_items(game, version)
+        for item in all_items:
+            if item.id == catid and item.type == cattype:
+                return item.data
+        return None

bemani/data/api/user.py

@@ -0,0 +1,292 @@
+import copy
+from typing import List, Tuple, Optional, Dict, Any
+
+from bemani.common import APIConstants, GameConstants, ValidatedDict, Parallel
+from bemani.data.interfaces import APIProviderInterface
+from bemani.data.api.base import BaseGlobalData
+from bemani.data.mysql.user import UserData
+from bemani.data.remoteuser import RemoteUser
+from bemani.data.types import UserID
+
+
+class GlobalUserData(BaseGlobalData):
+
+    def __init__(self, api: APIProviderInterface, user: UserData) -> None:
+        super().__init__(api)
+        self.user = user
+
+    def __format_ddr_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        updates = {}
+
+        area = profile.get_int('area', -1)
+        if area != -1:
+            updates['area'] = area
+
+        return updates
+
+    def __format_iidx_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        updates: Dict[str, Any] = {
+            'qpro': {},
+        }
+
+        area = profile.get_int('area', -1)
+        if area != -1:
+            updates['pid'] = area
+
+        qpro = profile.get_dict('qpro')
+        head = qpro.get_int('head', -1)
+        if head != -1:
+            updates['qpro']['head'] = head
+        hair = qpro.get_int('hair', -1)
+        if hair != -1:
+            updates['qpro']['hair'] = hair
+        face = qpro.get_int('face', -1)
+        if face != -1:
+            updates['qpro']['face'] = face
+        body = qpro.get_int('body', -1)
+        if body != -1:
+            updates['qpro']['body'] = body
+        hand = qpro.get_int('hand', -1)
+        if hand != -1:
+            updates['qpro']['hand'] = hand
+
+        return updates
+
+    def __format_jubeat_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        return {}
+
+    def __format_museca_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        return {}
+
+    def __format_popn_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        updates = {}
+
+        chara = profile.get_int('character', -1)
+        if chara != -1:
+            updates['chara'] = chara
+
+        return updates
+
+    def __format_reflec_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        updates = {}
+
+        icon = profile.get_int('icon', -1)
+        if icon != -1:
+            updates['config'] = {'icon_id': icon}
+
+        return updates
+
+    def __format_sdvx_profile(self, profile: ValidatedDict) -> Dict[str, Any]:
+        return {}
+
+    def __format_profile(self, profile: ValidatedDict) -> ValidatedDict:
+        base = {
+            'name': profile.get('name', ''),
+            'game': profile['game'],
+            'version': profile['version'],
+            'refid': profile['refid'],
+            'extid': profile['extid'],
+        }
+
+        if profile.get('game') == GameConstants.DDR:
+            base.update(self.__format_ddr_profile(profile))
+        if profile.get('game') == GameConstants.IIDX:
+            base.update(self.__format_iidx_profile(profile))
+        if profile.get('game') == GameConstants.JUBEAT:
+            base.update(self.__format_jubeat_profile(profile))
+        if profile.get('game') == GameConstants.MUSECA:
+            base.update(self.__format_museca_profile(profile))
+        if profile.get('game') == GameConstants.POPN_MUSIC:
+            base.update(self.__format_popn_profile(profile))
+        if profile.get('game') == GameConstants.REFLEC_BEAT:
+            base.update(self.__format_reflec_profile(profile))
+        if profile.get('game') == GameConstants.SDVX:
+            base.update(self.__format_sdvx_profile(profile))
+
+        return ValidatedDict(base)
+
+    def __profile_request(self, game: str, version: int, userid: UserID, exact: bool) -> Optional[ValidatedDict]:
+        # First, get or create the extid/refid for this virtual user
+        cardid = RemoteUser.userid_to_card(userid)
+        refid = self.user.get_refid(game, version, userid)
+        extid = self.user.get_extid(game, version, userid)
+
+        profiles = Parallel.flatten(Parallel.call(
+            [client.get_profiles for client in self.clients],
+            game,
+            version,
+            APIConstants.ID_TYPE_CARD,
+            [cardid],
+        ))
+        for profile in profiles:
+            cards = [card.upper() for card in profile.get('cards', [])]
+            if cardid in cards:
+                # Sanitize the returned data
+                profile = copy.deepcopy(profile)
+                del profile['cards']
+
+                exact_match = profile.get('match', 'partial') == 'exact'
+                if exact and (not exact_match):
+                    # This is a partial match, not for this game/version
+                    continue
+
+                if 'match' in profile:
+                    del profile['match']
+
+                # Add in our defaults we always provide
+                profile['game'] = game
+                profile['version'] = version if exact_match else 0
+                profile['refid'] = refid
+                profile['extid'] = extid
+
+                return self.__format_profile(ValidatedDict(profile))
+
+        return None
+
+    def from_cardid(self, cardid: str) -> Optional[UserID]:
+        userid = self.user.from_cardid(cardid)
+        if userid is None:
+            userid = RemoteUser.card_to_userid(cardid)
+        return userid
+
+    def from_refid(self, game: str, version: int, refid: str) -> Optional[UserID]:
+        return self.user.from_refid(game, version, refid)
+
+    def from_extid(self, game: str, version: int, extid: int) -> Optional[UserID]:
+        return self.user.from_extid(game, version, extid)
+
+    def get_profile(self, game: str, version: int, userid: UserID) -> Optional[ValidatedDict]:
+        if RemoteUser.is_remote(userid):
+            return self.__profile_request(game, version, userid, exact=True)
+        else:
+            return self.user.get_profile(game, version, userid)
+
+    def get_any_profile(self, game: str, version: int, userid: UserID) -> Optional[ValidatedDict]:
+        if RemoteUser.is_remote(userid):
+            return self.__profile_request(game, version, userid, exact=False)
+        else:
+            return self.user.get_any_profile(game, version, userid)
+
+    def get_any_profiles(self, game: str, version: int, userids: List[UserID]) -> List[Tuple[UserID, Optional[ValidatedDict]]]:
+        if len(userids) == 0:
+            return []
+
+        remote_ids = [
+            userid for userid in userids
+            if RemoteUser.is_remote(userid)
+        ]
+        local_ids = [
+            userid for userid in userids
+            if not RemoteUser.is_remote(userid)
+        ]
+
+        if len(remote_ids) == 0:
+            # We only have local profiles here, just pass on to the underlying layer
+            return self.user.get_any_profiles(game, version, local_ids)
+        else:
+            # We have to fetch some local profiles and some remote profiles, and then
+            # merge them together
+            card_to_userid = {
+                RemoteUser.userid_to_card(userid): userid
+                for userid in remote_ids
+            }
+
+            local_profiles, remote_profiles = Parallel.execute([
+                lambda: self.user.get_any_profiles(game, version, local_ids),
+                lambda: Parallel.flatten(Parallel.call(
+                    [client.get_profiles for client in self.clients],
+                    game,
+                    version,
+                    APIConstants.ID_TYPE_CARD,
+                    [RemoteUser.userid_to_card(userid) for userid in remote_ids],
+                ))
+            ])
+
+            for profile in remote_profiles:
+                cards = [card.upper() for card in profile.get('cards', [])]
+                for card in cards:
+                    # Map it back to the requested user
+                    userid = card_to_userid.get(card)
+                    if userid is None:
+                        continue
+
+                    # Sanitize the returned data
+                    profile = copy.deepcopy(profile)
+                    del profile['cards']
+
+                    exact_match = profile.get('match', 'partial') == 'exact'
+
+                    if 'match' in profile:
+                        del profile['match']
+
+                    refid = self.user.get_refid(game, version, userid)
+                    extid = self.user.get_extid(game, version, userid)
+
+                    # Add in our defaults we always provide
+                    profile['game'] = game
+                    profile['version'] = version if exact_match else 0
+                    profile['refid'] = refid
+                    profile['extid'] = extid
+
+                    local_profiles.append(
+                        (userid, self.__format_profile(ValidatedDict(profile))),
+                    )
+
+                    # Mark that we saw this card/user
+                    del card_to_userid[card]
+
+            # Finally, mark all missing remote profiles as None
+            for card in card_to_userid:
+                local_profiles.append((card_to_userid[card], None))
+
+            return local_profiles
+
+    def get_all_profiles(self, game: str, version: int) -> List[Tuple[UserID, ValidatedDict]]:
+        # Fetch local and remote profiles, and then merge by adding remote profiles to local
+        # profiles when we don't have a profile for that user ID yet.
+        local_cards, local_profiles, remote_profiles = Parallel.execute([
+            self.user.get_all_cards,
+            lambda: self.user.get_all_profiles(game, version),
+            lambda: Parallel.flatten(Parallel.call(
+                [client.get_profiles for client in self.clients],
+                game,
+                version,
+                APIConstants.ID_TYPE_SERVER,
+                [],
+            )),
+        ])
+
+        card_to_id = {cardid: userid for (cardid, userid) in local_cards}
+        id_to_profile = {userid: profile for (userid, profile) in local_profiles}
+
+        for profile in remote_profiles:
+            cardids = sorted([card.upper() for card in profile.get('cards', [])])
+            if len(cardids) == 0:
+                # We don't care about anonymous profiles
+                continue
+
+            local_cards = [cardid for cardid in cardids if cardid in card_to_id]
+            if len(local_cards) > 0:
+                # We have a local version of this profile!
+                continue
+
+            # Create a fake user with this profile
+            del profile['cards']
+
+            exact_match = profile.get('match', 'partial') == 'exact'
+            if not exact_match:
+                continue
+
+            userid = RemoteUser.card_to_userid(cardids[0])
+            refid = self.user.get_refid(game, version, userid)
+            extid = self.user.get_extid(game, version, userid)
+
+            # Add in our defaults we always provide
+            profile['game'] = game
+            profile['version'] = version
+            profile['refid'] = refid
+            profile['extid'] = extid
+
+            id_to_profile[userid] = self.__format_profile(ValidatedDict(profile))
+
+        return [(userid, id_to_profile[userid]) for userid in id_to_profile]