diff --git a/bemani/api/objects/catalog.py b/bemani/api/objects/catalog.py index aa6c34c..bc3953c 100644 --- a/bemani/api/objects/catalog.py +++ b/bemani/api/objects/catalog.py @@ -269,11 +269,6 @@ class CatalogObject(BaseObject): ) songs.extend(additions) - # Always a special case, of course. DanEvo has a virtual chart for play statistics - # tracking, so we need to filter that out here. - if self.game == GameConstants.DANCE_EVOLUTION: - songs = [song for song in songs if song.chart in [0, 1, 2, 3, 4]] - retval = { "songs": [self.__format_song(song) for song in songs], } diff --git a/bemani/api/objects/records.py b/bemani/api/objects/records.py index 51c30c8..6111dc4 100644 --- a/bemani/api/objects/records.py +++ b/bemani/api/objects/records.py @@ -330,13 +330,6 @@ class RecordsObject(BaseObject): if record.update >= until: continue - # Dance Evolution is a special case where it stores data in a virtual chart - # to keep track of play counts, due to the game not sending chart back with - # attempts. - if self.game == GameConstants.DANCE_EVOLUTION: - if record.chart not in [0, 1, 2, 3, 4]: - continue - if userid not in id_to_cards: cards = self.data.local.user.get_cards(userid) if len(cards) == 0: diff --git a/bemani/api/objects/statistics.py b/bemani/api/objects/statistics.py index 62886ab..bef0495 100644 --- a/bemani/api/objects/statistics.py +++ b/bemani/api/objects/statistics.py @@ -43,6 +43,7 @@ class StatisticsObject(BaseObject): GameConstants.JUBEAT, GameConstants.MUSECA, GameConstants.POPN_MUSIC, + GameConstants.DANCE_EVOLUTION, }: return True if self.game == GameConstants.IIDX: @@ -81,6 +82,8 @@ class StatisticsObject(BaseObject): DBConstants.SDVX_CLEAR_TYPE_NO_PLAY, DBConstants.SDVX_CLEAR_TYPE_FAILED, ] + if self.game == GameConstants.DANCE_EVOLUTION: + return attempt.data.get_int("grade") != DBConstants.DANEVO_GRADE_FAILED return False @@ -117,6 +120,8 @@ class StatisticsObject(BaseObject): DBConstants.SDVX_CLEAR_TYPE_ULTIMATE_CHAIN, DBConstants.SDVX_CLEAR_TYPE_PERFECT_ULTIMATE_CHAIN, ] + if self.game == GameConstants.DANCE_EVOLUTION: + return attempt.data.get_bool("full_combo") return False @@ -188,13 +193,6 @@ class StatisticsObject(BaseObject): def fetch_v1(self, idtype: APIConstants, ids: List[str], params: Dict[str, Any]) -> List[Dict[str, Any]]: retval: List[Dict[str, Any]] = [] - # Special case, Dance Evolution can't track attempts in any meaningful capacity - # because the game does not actually send down information about the chart for - # given attempts. So, we only know that players plays a song, not what chart or - # whether they cleared it or full-combo'd it. - if self.game == GameConstants.DANCE_EVOLUTION: - return [] - # Fetch the attempts if idtype == APIConstants.ID_TYPE_SERVER: retval = self.__aggregate_global( diff --git a/bemani/backend/danevo/base.py b/bemani/backend/danevo/base.py index 2cae6fb..19e3a08 100644 --- a/bemani/backend/danevo/base.py +++ b/bemani/backend/danevo/base.py @@ -20,7 +20,6 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base): CHART_TYPE_EXTREME: Final[int] = 2 CHART_TYPE_STEALTH: Final[int] = 3 CHART_TYPE_MASTER: Final[int] = 4 - CHART_TYPE_PLAYTRACKING: Final[int] = 5 GRADE_FAILED: Final[int] = DBConstants.DANEVO_GRADE_FAILED GRADE_E: Final[int] = DBConstants.DANEVO_GRADE_E @@ -41,6 +40,7 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base): def update_score( self, userid: UserID, + timestamp: int, songid: int, chart: int, points: int, @@ -79,24 +79,32 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base): chart, ) + history = ValidatedDict({}) + oldpoints = points + if oldscore is None: # If it is a new score, create a new dictionary to add to scoredata = ValidatedDict({}) highscore = True + raised = True else: # Set the score to any new record achieved highscore = points >= oldscore.points + raised = points > oldscore.points points = max(oldscore.points, points) scoredata = oldscore.data # Save combo scoredata.replace_int("combo", max(scoredata.get_int("combo"), combo)) + history.replace_int("combo", combo) # Save grade scoredata.replace_int("grade", max(scoredata.get_int("grade"), grade)) + history.replace_int("grade", grade) # Save full combo indicator. scoredata.replace_bool("full_combo", scoredata.get_bool("full_combo") or full_combo) + history.replace_bool("full_combo", full_combo) # Look up where this score was earned lid = self.get_machine_id() @@ -113,3 +121,17 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base): scoredata, highscore, ) + + # Save the history of this score too + self.data.local.music.put_attempt( + self.game, + self.version, + userid, + songid, + chart, + lid, + oldpoints, + history, + raised, + timestamp=timestamp, + ) diff --git a/bemani/backend/danevo/danevo.py b/bemani/backend/danevo/danevo.py index 2731a5d..cd18537 100644 --- a/bemani/backend/danevo/danevo.py +++ b/bemani/backend/danevo/danevo.py @@ -1,12 +1,11 @@ import base64 import struct -from typing import Any, Dict +from typing import Any, Dict, List from typing_extensions import Final from bemani.backend.ess import EventLogHandler from bemani.backend.danevo.base import DanceEvolutionBase from bemani.common import VersionConstants, Profile, CardCipher, Time -from bemani.data import ScoreSaveException from bemani.protocol import Node @@ -33,10 +32,9 @@ class DanceEvolution( # never sends trailing zeros for a data structure, even if it internally recognizes them, # so this may have length not divisible by 32 if there are no non-zero bytes after a certain # location. The correct thing to do is to fill with assumed zero bytes to a 32-byte boundary. - # The first 4 bytes of any history chunk are the score in little endian. The next byte is the - # song ID. Note that the chart does not appear anywhere in this structure to my knowledge. - # I have no idea what the rest of the bytes are, but most stay zeros and many are the same - # no matter what. + # The first 4 bytes of any history chunk are the score in little endian. The next 4 bytes + # is a packed structure containing the song ID, difficulty, score, grade, combo and full + # combo indicator. Then, a 64 bit timestamp in milliseconds follows. # DATA01-05 store the records for songs, including the high score, the letter grade earned, # and whether the song has been played (a record exists), cleared and whether a full combo @@ -384,11 +382,55 @@ class DanceEvolution( self.update_play_statistics(userid) # Now that we've got a fully updated profile to look at, let's see if we can't extract the last played songs - # and link those to records if they were a record. Unfortunately the game does not specify what chart was - # played in RDATA so we can't use that. We can, however, look at the song played positions in DATA03 and - # compare the score achieved to the various locations in DATAXX to see if it was a record thie time or not. + # and link those to records if they were a record. valid_ids = {song.id for song in self.data.local.music.get_all_songs(self.game, self.version)} + # Grab the last three record blobs from the RDATA chunk if it exists. + history: List[Dict[str, int]] = [] + + if "RDAT01" in usergamedata: + historydata = usergamedata["RDAT01"]["bindata"] + if len(historydata) < 3 * 32: + historydata += b"\x00" * ((3 * 32) - len(historydata)) + + for offset in range(0, 32 * 3, 32): + score, params, ts = struct.unpack("> 8) & 0xF) + if chart is None: + continue + + grade = { + self.GAME_GRADE_FAILED: self.GRADE_FAILED, + self.GAME_GRADE_E: self.GRADE_E, + self.GAME_GRADE_D: self.GRADE_D, + self.GAME_GRADE_C: self.GRADE_C, + self.GAME_GRADE_B: self.GRADE_B, + self.GAME_GRADE_A: self.GRADE_A, + self.GAME_GRADE_AA: self.GRADE_AA, + self.GAME_GRADE_AAA: self.GRADE_AAA, + }[(params >> 27) & 0x7] + + history.append( + { + "id": params & 0xFF, + "chart": chart, + "score": score, + "grade": grade, + "combo": (params >> 12) & 0x3FF, + "fc": (params >> 30) & 0x3, + "ts": ts // 1000, + } + ) + if "DATA03" in usergamedata: strdatalist = usergamedata["DATA03"]["strdata"].split(b",") @@ -404,55 +446,29 @@ class DanceEvolution( strdatalist[self.DATA03_THIRD_HIGH_SCORE_OFFSET].decode("shift-jis").split(".")[0] ) - for played, scored in [ - (first_song_played, first_song_scored), - (second_song_played, second_song_scored), - (third_song_played, third_song_scored), + songcount = 0 + for possible in [first_song_played, second_song_played, third_song_played]: + if possible in valid_ids: + songcount += 1 + + # Scores are from newest to oldest. + history = list(reversed(history[:songcount])) + + for stage, played, scored in [ + (0, first_song_played, first_song_scored), + (1, second_song_played, second_song_scored), + (2, third_song_played, third_song_scored), ]: if played not in valid_ids: # Game might be set to 1 song. continue - # For the purpose of popularity tracking, save an attempt for this song into the virtual - # attempt chart, since we can't know for certain what chart an attempt was associated with. - now = Time.now() - lid = self.get_machine_id() - - for bump in range(10): - timestamp = now + bump - - self.data.local.music.put_score( - self.game, - self.version, - userid, - played, - self.CHART_TYPE_PLAYTRACKING, - lid, - scored, - {}, - False, - timestamp=timestamp, - ) - - try: - self.data.local.music.put_attempt( - self.game, - self.version, - userid, - played, - self.CHART_TYPE_PLAYTRACKING, - lid, - scored, - {}, - False, - timestamp=timestamp, - ) - except ScoreSaveException: - # Try again one second in the future - continue - - # We saved successfully - break + # Attempt to find the play in our extracted attempts. + if history[stage]["id"] != played: + continue + if history[stage]["score"] != scored: + continue + attempt = history[stage] # First, calculate whether we're going to look at DATA01-05 or DATA11-15. if played < 63: @@ -481,6 +497,10 @@ class DanceEvolution( # because it couldn't possibly be it. continue + if mapping[key] != attempt["chart"]: + # This isn't the right chart for what was played. + continue + # They could have played some other songs and the game truncated this because it # only ever sends back until the last nonzero value, so we need to fill in the blanks # so to speak with all zero bytes. @@ -489,29 +509,20 @@ class DanceEvolution( chunk = chunk + (b"\x00" * (8 - len(chunk))) record, combo, playmarker, _, stats = struct.unpack("> 1) & 0x7 - grade = { - self.GAME_GRADE_FAILED: self.GRADE_FAILED, - self.GAME_GRADE_E: self.GRADE_E, - self.GAME_GRADE_D: self.GRADE_D, - self.GAME_GRADE_C: self.GRADE_C, - self.GAME_GRADE_B: self.GRADE_B, - self.GAME_GRADE_A: self.GRADE_A, - self.GAME_GRADE_AA: self.GRADE_AA, - self.GAME_GRADE_AAA: self.GRADE_AAA, - }[letter_grade] - - self.update_score(userid, played, mapping[key], scored, grade, combo, full_combo) + self.update_score( + userid, + attempt["ts"], + attempt["id"], + attempt["chart"], + scored, + attempt["grade"], + attempt["combo"], + attempt["fc"] != 0, + ) # Don't need to update anything else now. break diff --git a/bemani/data/migrations/versions/4fbae3d0cb31_remove_bad_danevo_charts_from_the_db.py b/bemani/data/migrations/versions/4fbae3d0cb31_remove_bad_danevo_charts_from_the_db.py new file mode 100644 index 0000000..cd990c3 --- /dev/null +++ b/bemani/data/migrations/versions/4fbae3d0cb31_remove_bad_danevo_charts_from_the_db.py @@ -0,0 +1,37 @@ +"""Remove bad DanEvo charts from the DB. + +Revision ID: 4fbae3d0cb31 +Revises: f64d138962e0 +Create Date: 2025-10-24 00:59:53.007135 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import text + + +# revision identifiers, used by Alembic. +revision = '4fbae3d0cb31' +down_revision = 'f64d138962e0' +branch_labels = None +depends_on = None + + +def upgrade(): + conn = op.get_bind() + + # Remove any attempts that were made on now-invalid charts. + sql = "DELETE FROM score WHERE musicid IN (SELECT id FROM music WHERE game = 'danevo' AND chart = 5)" + conn.execute(text(sql), {}) + + sql = "DELETE FROM score_history WHERE musicid IN (SELECT id FROM music WHERE game = 'danevo' AND chart = 5)" + conn.execute(text(sql), {}) + + sql = "DELETE FROM music WHERE game = 'danevo' AND chart = 5" + conn.execute(text(sql), {}) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/bemani/frontend/app.py b/bemani/frontend/app.py index 10eff99..62cd703 100644 --- a/bemani/frontend/app.py +++ b/bemani/frontend/app.py @@ -430,6 +430,10 @@ def navigation() -> Dict[str, Any]: "label": "Dance Mates", "uri": url_for("danevo_pages.viewdancemates", userid=g.userID), }, + { + "label": "Personal Scores", + "uri": url_for("danevo_pages.viewscores", userid=g.userID), + }, { "label": "Personal Records", "uri": url_for("danevo_pages.viewrecords", userid=g.userID), @@ -438,6 +442,10 @@ def navigation() -> Dict[str, Any]: ) danevo_entries.extend( [ + { + "label": "Global Scores", + "uri": url_for("danevo_pages.viewnetworkscores"), + }, { "label": "Global Records", "uri": url_for("danevo_pages.viewnetworkrecords"), diff --git a/bemani/frontend/danevo/danevo.py b/bemani/frontend/danevo/danevo.py index 9fcc6df..5c7b13e 100644 --- a/bemani/frontend/danevo/danevo.py +++ b/bemani/frontend/danevo/danevo.py @@ -16,8 +16,6 @@ class DanceEvolutionFrontend(FrontendBase): DanceEvolutionBase.CHART_TYPE_EXTREME, DanceEvolutionBase.CHART_TYPE_STEALTH, DanceEvolutionBase.CHART_TYPE_MASTER, - # Only included so that we can grab the play count for this song. - DanceEvolutionBase.CHART_TYPE_PLAYTRACKING, ] valid_rival_types: List[str] = ["dancemate"] @@ -60,7 +58,21 @@ class DanceEvolutionFrontend(FrontendBase): return formatted_score def format_attempt(self, userid: UserID, attempt: Attempt) -> Dict[str, Any]: - raise NotImplementedError("Dance Evolution does not have attempts!") + formatted_attempt = super().format_attempt(userid, attempt) + formatted_attempt["combo"] = attempt.data.get_int("combo") + formatted_attempt["full_combo"] = attempt.data.get_bool("full_combo") + formatted_attempt["medal"] = attempt.data.get_int("grade") + formatted_attempt["grade"] = { + DanceEvolutionBase.GRADE_FAILED: "FAILED", + DanceEvolutionBase.GRADE_E: "E", + DanceEvolutionBase.GRADE_D: "D", + DanceEvolutionBase.GRADE_C: "C", + DanceEvolutionBase.GRADE_B: "B", + DanceEvolutionBase.GRADE_A: "A", + DanceEvolutionBase.GRADE_AA: "AA", + DanceEvolutionBase.GRADE_AAA: "AAA", + }.get(attempt.data.get_int("grade"), "NO PLAY") + return formatted_attempt def format_profile(self, profile: Profile, playstats: ValidatedDict) -> Dict[str, Any]: formatted_profile = super().format_profile(profile, playstats) @@ -80,9 +92,6 @@ class DanceEvolutionFrontend(FrontendBase): return formatted_song def merge_song(self, existing: Dict[str, Any], new: Song) -> Dict[str, Any]: - if new.chart == DanceEvolutionBase.CHART_TYPE_PLAYTRACKING: - return existing - new_song = super().merge_song(existing, new) if existing["levels"][new.chart] == 0: new_song["levels"][new.chart] = new.data.get_int("level") diff --git a/bemani/frontend/danevo/endpoints.py b/bemani/frontend/danevo/endpoints.py index e2232b3..efe4df1 100644 --- a/bemani/frontend/danevo/endpoints.py +++ b/bemani/frontend/danevo/endpoints.py @@ -20,6 +20,84 @@ danevo_pages = Blueprint( ) +@danevo_pages.route("/scores") +@loginrequired +def viewnetworkscores() -> Response: + # Only load the last 100 results for the initial fetch, so we can render faster + frontend = DanceEvolutionFrontend(g.data, g.config, g.cache) + network_scores = frontend.get_network_scores(limit=100) + if len(network_scores["attempts"]) > 10: + network_scores["attempts"] = frontend.round_to_ten(network_scores["attempts"]) + + return render_react( + "Global Dance Evolution Scores", + "danevo/scores.react.js", + { + "attempts": network_scores["attempts"], + "songs": frontend.get_all_songs(), + "players": network_scores["players"], + "versions": {version: name for (game, version, name) in frontend.all_games()}, + "shownames": True, + "shownewrecords": False, + }, + { + "refresh": url_for("danevo_pages.listnetworkscores"), + "player": url_for("danevo_pages.viewplayer", userid=-1), + "individual_score": url_for("danevo_pages.viewtopscores", musicid=-1), + }, + ) + + +@danevo_pages.route("/scores/list") +@jsonify +@loginrequired +def listnetworkscores() -> Dict[str, Any]: + frontend = DanceEvolutionFrontend(g.data, g.config, g.cache) + return frontend.get_network_scores() + + +@danevo_pages.route("/scores/") +@loginrequired +def viewscores(userid: UserID) -> Response: + frontend = DanceEvolutionFrontend(g.data, g.config, g.cache) + info = frontend.get_latest_player_info([userid]).get(userid) + if info is None: + abort(404) + + scores = frontend.get_scores(userid, limit=100) + if len(scores) > 10: + scores = frontend.round_to_ten(scores) + + return render_react( + f'{info["name"]}\'s Dance Evolution Scores', + "danevo/scores.react.js", + { + "attempts": scores, + "songs": frontend.get_all_songs(), + "players": {}, + "versions": {version: name for (game, version, name) in frontend.all_games()}, + "shownames": False, + "shownewrecords": True, + }, + { + "refresh": url_for("danevo_pages.listscores", userid=userid), + "player": url_for("danevo_pages.viewplayer", userid=-1), + "individual_score": url_for("danevo_pages.viewtopscores", musicid=-1), + }, + ) + + +@danevo_pages.route("/scores//list") +@jsonify +@loginrequired +def listscores(userid: UserID) -> Dict[str, Any]: + frontend = DanceEvolutionFrontend(g.data, g.config, g.cache) + return { + "attempts": frontend.get_scores(userid), + "players": {}, + } + + @danevo_pages.route("/records") @loginrequired def viewnetworkrecords() -> Response: diff --git a/bemani/frontend/static/controllers/danevo/records.react.js b/bemani/frontend/static/controllers/danevo/records.react.js index 344251a..e32fb46 100644 --- a/bemani/frontend/static/controllers/danevo/records.react.js +++ b/bemani/frontend/static/controllers/danevo/records.react.js @@ -129,10 +129,9 @@ var network_records = createReactClass({ getPlays: function(record) { if (!record) { return 0; } var plays = 0; - - // Play counts are only storted in the play statistics chart. - if (record[5]) { plays += record[5].plays; } - + for (var i = 0; i < 5; i++) { + if (record[i]) { plays += record[i].plays; } + } return plays; }, @@ -468,7 +467,7 @@ var network_records = createReactClass({ - + { this.state.offset > 0 ? +
+ Score + {score.points} + Grade + {score.grade} +
+
+ Combo + {score.combo <= 0 ? '-' : score.combo} + { score.full_combo ? full combo : null } +
+ + ); + }, + + render: function() { + return ( +
+ + + + { window.shownames ? : null } + + + + + + + + {this.state.attempts.map(function(attempt, index) { + if (index < this.state.offset || index >= this.state.offset + this.state.limit) { + return null; + } + + return ( + + { window.shownames ? : null } + + + + + + ); + }.bind(this))} + + + + + + +
NameTimestampSong / ArtistDifficultyScore
{ + this.state.players[attempt.userid].name + } +
+ + { window.shownewrecords && attempt.raised ? + new high score! : + null + } +
+ 		+ +
{ this.state.songs[attempt.songid].name }
+
{ this.state.songs[attempt.songid].artist }
+ + 		+ + { this.renderScore(attempt) }
+ { this.state.offset > 0 ? + : null + } + { (this.state.offset + this.state.limit) < this.state.attempts.length ? + = this.state.attempts.length) { return } + this.setState({offset: page}); + }.bind(this)}/> : + this.state.loading ? + + loading more scores... + : null + } +
+
+ ); + }, +}); + +ReactDOM.render( + React.createElement(network_scores, null), + document.getElementById('content') +); diff --git a/bemani/utils/read.py b/bemani/utils/read.py index cd56df2..6e37dfa 100644 --- a/bemani/utils/read.py +++ b/bemani/utils/read.py @@ -6186,7 +6186,7 @@ class ImportDanceEvolution(ImportBase): # Import it self.start_batch() - for chart_id in [0, 1, 2, 3, 4, 5]: + for chart_id in [0, 1, 2, 3, 4]: # First, try to find in the DB from another version old_id = self.get_music_id_for_song(song["id"], chart_id) if self.no_combine or old_id is None: