Leahnaya/bemaniutils
1
0
Fork 0
mirror of https://github.com/DragonMinded/bemaniutils.git synced 2026-03-21 17:24:33 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make use of information about DanEvo attempts to correctly track attempts on the network.

Browse Source
This commit is contained in:
Jennifer Taylor 2025-10-24 01:33:04 +00:00
parent df4a9c712d
commit 508670b0fc
12 changed files with 411 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bemani/api/objects/catalog.py
View File

@ -269,11 +269,6 @@ class CatalogObject(BaseObject):
)
songs.extend(additions)
# Always a special case, of course. DanEvo has a virtual chart for play statistics
# tracking, so we need to filter that out here.
if self.game == GameConstants.DANCE_EVOLUTION:
songs = [song for song in songs if song.chart in [0, 1, 2, 3, 4]]
retval = {
"songs": [self.__format_song(song) for song in songs],
}

7
bemani/api/objects/records.py
View File

@ -330,13 +330,6 @@ class RecordsObject(BaseObject):
if record.update >= until:
continue
# Dance Evolution is a special case where it stores data in a virtual chart
# to keep track of play counts, due to the game not sending chart back with
# attempts.
if self.game == GameConstants.DANCE_EVOLUTION:
if record.chart not in [0, 1, 2, 3, 4]:
continue
if userid not in id_to_cards:
cards = self.data.local.user.get_cards(userid)
if len(cards) == 0:

12
bemani/api/objects/statistics.py
View File

@ -43,6 +43,7 @@ class StatisticsObject(BaseObject):
GameConstants.JUBEAT,
GameConstants.MUSECA,
GameConstants.POPN_MUSIC,
GameConstants.DANCE_EVOLUTION,
}:
return True
if self.game == GameConstants.IIDX:
@ -81,6 +82,8 @@ class StatisticsObject(BaseObject):
DBConstants.SDVX_CLEAR_TYPE_NO_PLAY,
DBConstants.SDVX_CLEAR_TYPE_FAILED,
]
if self.game == GameConstants.DANCE_EVOLUTION:
return attempt.data.get_int("grade") != DBConstants.DANEVO_GRADE_FAILED
return False
@ -117,6 +120,8 @@ class StatisticsObject(BaseObject):
DBConstants.SDVX_CLEAR_TYPE_ULTIMATE_CHAIN,
DBConstants.SDVX_CLEAR_TYPE_PERFECT_ULTIMATE_CHAIN,
]
if self.game == GameConstants.DANCE_EVOLUTION:
return attempt.data.get_bool("full_combo")
return False
@ -188,13 +193,6 @@ class StatisticsObject(BaseObject):
def fetch_v1(self, idtype: APIConstants, ids: List[str], params: Dict[str, Any]) -> List[Dict[str, Any]]:
retval: List[Dict[str, Any]] = []
# Special case, Dance Evolution can't track attempts in any meaningful capacity
# because the game does not actually send down information about the chart for
# given attempts. So, we only know that players plays a song, not what chart or
# whether they cleared it or full-combo'd it.
if self.game == GameConstants.DANCE_EVOLUTION:
return []
# Fetch the attempts
if idtype == APIConstants.ID_TYPE_SERVER:
retval = self.__aggregate_global(

24
bemani/backend/danevo/base.py
View File

@ -20,7 +20,6 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
CHART_TYPE_EXTREME: Final[int] = 2
CHART_TYPE_STEALTH: Final[int] = 3
CHART_TYPE_MASTER: Final[int] = 4
CHART_TYPE_PLAYTRACKING: Final[int] = 5
GRADE_FAILED: Final[int] = DBConstants.DANEVO_GRADE_FAILED
GRADE_E: Final[int] = DBConstants.DANEVO_GRADE_E
@ -41,6 +40,7 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
def update_score(
self,
userid: UserID,
timestamp: int,
songid: int,
chart: int,
points: int,
@ -79,24 +79,32 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
chart,
)
history = ValidatedDict({})
oldpoints = points
if oldscore is None:
# If it is a new score, create a new dictionary to add to
scoredata = ValidatedDict({})
highscore = True
raised = True
else:
# Set the score to any new record achieved
highscore = points >= oldscore.points
raised = points > oldscore.points
points = max(oldscore.points, points)
scoredata = oldscore.data
# Save combo
scoredata.replace_int("combo", max(scoredata.get_int("combo"), combo))
history.replace_int("combo", combo)
# Save grade
scoredata.replace_int("grade", max(scoredata.get_int("grade"), grade))
history.replace_int("grade", grade)
# Save full combo indicator.
scoredata.replace_bool("full_combo", scoredata.get_bool("full_combo") or full_combo)
history.replace_bool("full_combo", full_combo)
# Look up where this score was earned
lid = self.get_machine_id()
@ -113,3 +121,17 @@ class DanceEvolutionBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
scoredata,
highscore,
)
# Save the history of this score too
self.data.local.music.put_attempt(
self.game,
self.version,
userid,
songid,
chart,
lid,
oldpoints,
history,
raised,
timestamp=timestamp,
)

155
bemani/backend/danevo/danevo.py
View File

@ -1,12 +1,11 @@
import base64
import struct
from typing import Any, Dict
from typing import Any, Dict, List
from typing_extensions import Final
from bemani.backend.ess import EventLogHandler
from bemani.backend.danevo.base import DanceEvolutionBase
from bemani.common import VersionConstants, Profile, CardCipher, Time
from bemani.data import ScoreSaveException
from bemani.protocol import Node
@ -33,10 +32,9 @@ class DanceEvolution(
# never sends trailing zeros for a data structure, even if it internally recognizes them,
# so this may have length not divisible by 32 if there are no non-zero bytes after a certain
# location. The correct thing to do is to fill with assumed zero bytes to a 32-byte boundary.
# The first 4 bytes of any history chunk are the score in little endian. The next byte is the
# song ID. Note that the chart does not appear anywhere in this structure to my knowledge.
# I have no idea what the rest of the bytes are, but most stay zeros and many are the same
# no matter what.
# The first 4 bytes of any history chunk are the score in little endian. The next 4 bytes
# is a packed structure containing the song ID, difficulty, score, grade, combo and full
# combo indicator. Then, a 64 bit timestamp in milliseconds follows.
# DATA01-05 store the records for songs, including the high score, the letter grade earned,
# and whether the song has been played (a record exists), cleared and whether a full combo
@ -384,11 +382,55 @@ class DanceEvolution(
self.update_play_statistics(userid)
# Now that we've got a fully updated profile to look at, let's see if we can't extract the last played songs
# and link those to records if they were a record. Unfortunately the game does not specify what chart was
# played in RDATA so we can't use that. We can, however, look at the song played positions in DATA03 and
# compare the score achieved to the various locations in DATAXX to see if it was a record thie time or not.
# and link those to records if they were a record.
valid_ids = {song.id for song in self.data.local.music.get_all_songs(self.game, self.version)}
# Grab the last three record blobs from the RDATA chunk if it exists.
history: List[Dict[str, int]] = []
if "RDAT01" in usergamedata:
historydata = usergamedata["RDAT01"]["bindata"]
if len(historydata) < 3 * 32:
historydata += b"\x00" * ((3 * 32) - len(historydata))
for offset in range(0, 32 * 3, 32):
score, params, ts = struct.unpack("<IIQ", historydata[offset : (offset + 16)])
if not score and not params and not ts:
continue
chart = {
0: self.CHART_TYPE_LIGHT,
1: self.CHART_TYPE_STANDARD,
2: self.CHART_TYPE_EXTREME,
3: self.CHART_TYPE_STEALTH,
4: self.CHART_TYPE_MASTER,
}.get((params >> 8) & 0xF)
if chart is None:
continue
grade = {
self.GAME_GRADE_FAILED: self.GRADE_FAILED,
self.GAME_GRADE_E: self.GRADE_E,
self.GAME_GRADE_D: self.GRADE_D,
self.GAME_GRADE_C: self.GRADE_C,
self.GAME_GRADE_B: self.GRADE_B,
self.GAME_GRADE_A: self.GRADE_A,
self.GAME_GRADE_AA: self.GRADE_AA,
self.GAME_GRADE_AAA: self.GRADE_AAA,
}[(params >> 27) & 0x7]
history.append(
{
"id": params & 0xFF,
"chart": chart,
"score": score,
"grade": grade,
"combo": (params >> 12) & 0x3FF,
"fc": (params >> 30) & 0x3,
"ts": ts // 1000,
}
)
if "DATA03" in usergamedata:
strdatalist = usergamedata["DATA03"]["strdata"].split(b",")
@ -404,55 +446,29 @@ class DanceEvolution(
strdatalist[self.DATA03_THIRD_HIGH_SCORE_OFFSET].decode("shift-jis").split(".")[0]
)
for played, scored in [
(first_song_played, first_song_scored),
(second_song_played, second_song_scored),
(third_song_played, third_song_scored),
songcount = 0
for possible in [first_song_played, second_song_played, third_song_played]:
if possible in valid_ids:
songcount += 1
# Scores are from newest to oldest.
history = list(reversed(history[:songcount]))
for stage, played, scored in [
(0, first_song_played, first_song_scored),
(1, second_song_played, second_song_scored),
(2, third_song_played, third_song_scored),
]:
if played not in valid_ids:
# Game might be set to 1 song.
continue
# For the purpose of popularity tracking, save an attempt for this song into the virtual
# attempt chart, since we can't know for certain what chart an attempt was associated with.
now = Time.now()
lid = self.get_machine_id()
for bump in range(10):
timestamp = now + bump
self.data.local.music.put_score(
self.game,
self.version,
userid,
played,
self.CHART_TYPE_PLAYTRACKING,
lid,
scored,
{},
False,
timestamp=timestamp,
)
try:
self.data.local.music.put_attempt(
self.game,
self.version,
userid,
played,
self.CHART_TYPE_PLAYTRACKING,
lid,
scored,
{},
False,
timestamp=timestamp,
)
except ScoreSaveException:
# Try again one second in the future
continue
# We saved successfully
break
# Attempt to find the play in our extracted attempts.
if history[stage]["id"] != played:
continue
if history[stage]["score"] != scored:
continue
attempt = history[stage]
# First, calculate whether we're going to look at DATA01-05 or DATA11-15.
if played < 63:
@ -481,6 +497,10 @@ class DanceEvolution(
# because it couldn't possibly be it.
continue
if mapping[key] != attempt["chart"]:
# This isn't the right chart for what was played.
continue
# They could have played some other songs and the game truncated this because it
# only ever sends back until the last nonzero value, so we need to fill in the blanks
# so to speak with all zero bytes.
@ -489,29 +509,20 @@ class DanceEvolution(
chunk = chunk + (b"\x00" * (8 - len(chunk)))
record, combo, playmarker, _, stats = struct.unpack("<Ibbbb", chunk)
if record != scored:
# This wasn't the record we just earned.
continue
if not playmarker:
# This wasn't actually played.
continue
# Found it!
full_combo = bool(stats & 0x10)
letter_grade = (stats >> 1) & 0x7
grade = {
self.GAME_GRADE_FAILED: self.GRADE_FAILED,
self.GAME_GRADE_E: self.GRADE_E,
self.GAME_GRADE_D: self.GRADE_D,
self.GAME_GRADE_C: self.GRADE_C,
self.GAME_GRADE_B: self.GRADE_B,
self.GAME_GRADE_A: self.GRADE_A,
self.GAME_GRADE_AA: self.GRADE_AA,
self.GAME_GRADE_AAA: self.GRADE_AAA,
}[letter_grade]
self.update_score(userid, played, mapping[key], scored, grade, combo, full_combo)
self.update_score(
userid,
attempt["ts"],
attempt["id"],
attempt["chart"],
scored,
attempt["grade"],
attempt["combo"],
attempt["fc"] != 0,
)
# Don't need to update anything else now.
break

37
bemani/data/migrations/versions/4fbae3d0cb31_remove_bad_danevo_charts_from_the_db.py Normal file
View File

@ -0,0 +1,37 @@
"""Remove bad DanEvo charts from the DB.
Revision ID: 4fbae3d0cb31
Revises: f64d138962e0
Create Date: 2025-10-24 00:59:53.007135
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.sql import text
# revision identifiers, used by Alembic.
revision = '4fbae3d0cb31'
down_revision = 'f64d138962e0'
branch_labels = None
depends_on = None
def upgrade():
conn = op.get_bind()
# Remove any attempts that were made on now-invalid charts.
sql = "DELETE FROM score WHERE musicid IN (SELECT id FROM music WHERE game = 'danevo' AND chart = 5)"
conn.execute(text(sql), {})
sql = "DELETE FROM score_history WHERE musicid IN (SELECT id FROM music WHERE game = 'danevo' AND chart = 5)"
conn.execute(text(sql), {})
sql = "DELETE FROM music WHERE game = 'danevo' AND chart = 5"
conn.execute(text(sql), {})
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

8
bemani/frontend/app.py
View File

@ -430,6 +430,10 @@ def navigation() -> Dict[str, Any]:
"label": "Dance Mates",
"uri": url_for("danevo_pages.viewdancemates", userid=g.userID),
},
{
"label": "Personal Scores",
"uri": url_for("danevo_pages.viewscores", userid=g.userID),
},
{
"label": "Personal Records",
"uri": url_for("danevo_pages.viewrecords", userid=g.userID),
@ -438,6 +442,10 @@ def navigation() -> Dict[str, Any]:
)
danevo_entries.extend(
[
{
"label": "Global Scores",
"uri": url_for("danevo_pages.viewnetworkscores"),
},
{
"label": "Global Records",
"uri": url_for("danevo_pages.viewnetworkrecords"),

21
bemani/frontend/danevo/danevo.py
View File

@ -16,8 +16,6 @@ class DanceEvolutionFrontend(FrontendBase):
DanceEvolutionBase.CHART_TYPE_EXTREME,
DanceEvolutionBase.CHART_TYPE_STEALTH,
DanceEvolutionBase.CHART_TYPE_MASTER,
# Only included so that we can grab the play count for this song.
DanceEvolutionBase.CHART_TYPE_PLAYTRACKING,
]
valid_rival_types: List[str] = ["dancemate"]
@ -60,7 +58,21 @@ class DanceEvolutionFrontend(FrontendBase):
return formatted_score
def format_attempt(self, userid: UserID, attempt: Attempt) -> Dict[str, Any]:
raise NotImplementedError("Dance Evolution does not have attempts!")
formatted_attempt = super().format_attempt(userid, attempt)
formatted_attempt["combo"] = attempt.data.get_int("combo")
formatted_attempt["full_combo"] = attempt.data.get_bool("full_combo")
formatted_attempt["medal"] = attempt.data.get_int("grade")
formatted_attempt["grade"] = {
DanceEvolutionBase.GRADE_FAILED: "FAILED",
DanceEvolutionBase.GRADE_E: "E",
DanceEvolutionBase.GRADE_D: "D",
DanceEvolutionBase.GRADE_C: "C",
DanceEvolutionBase.GRADE_B: "B",
DanceEvolutionBase.GRADE_A: "A",
DanceEvolutionBase.GRADE_AA: "AA",
DanceEvolutionBase.GRADE_AAA: "AAA",
}.get(attempt.data.get_int("grade"), "NO PLAY")
return formatted_attempt
def format_profile(self, profile: Profile, playstats: ValidatedDict) -> Dict[str, Any]:
formatted_profile = super().format_profile(profile, playstats)
@ -80,9 +92,6 @@ class DanceEvolutionFrontend(FrontendBase):
return formatted_song
def merge_song(self, existing: Dict[str, Any], new: Song) -> Dict[str, Any]:
if new.chart == DanceEvolutionBase.CHART_TYPE_PLAYTRACKING:
return existing
new_song = super().merge_song(existing, new)
if existing["levels"][new.chart] == 0:
new_song["levels"][new.chart] = new.data.get_int("level")

78
bemani/frontend/danevo/endpoints.py
View File

@ -20,6 +20,84 @@ danevo_pages = Blueprint(
)
@danevo_pages.route("/scores")
@loginrequired
def viewnetworkscores() -> Response:
# Only load the last 100 results for the initial fetch, so we can render faster
frontend = DanceEvolutionFrontend(g.data, g.config, g.cache)
network_scores = frontend.get_network_scores(limit=100)
if len(network_scores["attempts"]) > 10:
network_scores["attempts"] = frontend.round_to_ten(network_scores["attempts"])
return render_react(
"Global Dance Evolution Scores",
"danevo/scores.react.js",
{
"attempts": network_scores["attempts"],
"songs": frontend.get_all_songs(),
"players": network_scores["players"],
"versions": {version: name for (game, version, name) in frontend.all_games()},
"shownames": True,
"shownewrecords": False,
},
{
"refresh": url_for("danevo_pages.listnetworkscores"),
"player": url_for("danevo_pages.viewplayer", userid=-1),
"individual_score": url_for("danevo_pages.viewtopscores", musicid=-1),
},
)
@danevo_pages.route("/scores/list")
@jsonify
@loginrequired
def listnetworkscores() -> Dict[str, Any]:
frontend = DanceEvolutionFrontend(g.data, g.config, g.cache)
return frontend.get_network_scores()
@danevo_pages.route("/scores/<int:userid>")
@loginrequired
def viewscores(userid: UserID) -> Response:
frontend = DanceEvolutionFrontend(g.data, g.config, g.cache)
info = frontend.get_latest_player_info([userid]).get(userid)
if info is None:
abort(404)
scores = frontend.get_scores(userid, limit=100)
if len(scores) > 10:
scores = frontend.round_to_ten(scores)
return render_react(
f'{info["name"]}\'s Dance Evolution Scores',
"danevo/scores.react.js",
{
"attempts": scores,
"songs": frontend.get_all_songs(),
"players": {},
"versions": {version: name for (game, version, name) in frontend.all_games()},
"shownames": False,
"shownewrecords": True,
},
{
"refresh": url_for("danevo_pages.listscores", userid=userid),
"player": url_for("danevo_pages.viewplayer", userid=-1),
"individual_score": url_for("danevo_pages.viewtopscores", musicid=-1),
},
)
@danevo_pages.route("/scores/<int:userid>/list")
@jsonify
@loginrequired
def listscores(userid: UserID) -> Dict[str, Any]:
frontend = DanceEvolutionFrontend(g.data, g.config, g.cache)
return {
"attempts": frontend.get_scores(userid),
"players": {},
}
@danevo_pages.route("/records")
@loginrequired
def viewnetworkrecords() -> Response:

9
bemani/frontend/static/controllers/danevo/records.react.js
View File

@ -129,10 +129,9 @@ var network_records = createReactClass({
getPlays: function(record) {
if (!record) { return 0; }
var plays = 0;
// Play counts are only storted in the play statistics chart.
if (record[5]) { plays += record[5].plays; }
for (var i = 0; i < 5; i++) {
if (record[i]) { plays += record[i].plays; }
}
return plays;
},
@ -468,7 +467,7 @@ var network_records = createReactClass({
</tbody>
<tfoot>
<tr>
<td colSpan={5}>
<td colSpan={6}>
{ this.state.offset > 0 ?
<Prev onClick={function(event) {
var page = this.state.offset - this.state.limit;

157
bemani/frontend/static/controllers/danevo/scores.react.js Normal file
View File

@ -0,0 +1,157 @@
/*** @jsx React.DOM */
var network_scores = createReactClass({
getInitialState: function(props) {
return {
songs: window.songs,
attempts: window.attempts,
players: window.players,
versions: window.versions,
loading: true,
offset: 0,
limit: 10,
};
},
componentDidMount: function() {
this.refreshScores();
},
refreshScores: function() {
AJAX.get(
Link.get('refresh'),
function(response) {
this.setState({
attempts: response.attempts,
players: response.players,
loading: false,
});
// Refresh every 15 seconds
setTimeout(this.refreshScores, 15000);
}.bind(this)
);
},
convertChart: function(chart) {
switch(chart) {
case 0:
return 'Light';
case 1:
return 'Standard';
case 2:
return 'Extreme';
case 3:
return 'Stealth';
case 4:
return 'Master';
default:
return 'u broke it';
}
},
renderScore: function(score) {
return (
<div className="score">
<div>
<span className="label">Score</span>
<span className="score">{score.points}</span>
<span className="label">Grade</span>
<span className="score">{score.grade}</span>
</div>
<div>
<span className="label">Combo</span>
<span className="score">{score.combo <= 0 ? '-' : score.combo}</span>
{ score.full_combo ? <span className="label">full combo</span> : null }
</div>
</div>
);
},
render: function() {
return (
<div>
<table className="list attempts">
<thead>
<tr>
{ window.shownames ? <th>Name</th> : null }
<th>Timestamp</th>
<th>Song / Artist</th>
<th>Difficulty</th>
<th>Score</th>
</tr>
</thead>
<tbody>
{this.state.attempts.map(function(attempt, index) {
if (index < this.state.offset || index >= this.state.offset + this.state.limit) {
return null;
}
return (
<tr>
{ window.shownames ? <td><a href={Link.get('player', attempt.userid)}>{
this.state.players[attempt.userid].name
}</a></td> : null }
<td>
<div>
<Timestamp timestamp={attempt.timestamp} />
{ window.shownewrecords && attempt.raised ?
<span className="raised">new high score!</span> :
null
}
</div>
</td>
<td className="center">
<a href={Link.get('individual_score', attempt.songid)}>
<div className="songname">{ this.state.songs[attempt.songid].name }</div>
<div className="songartist">{ this.state.songs[attempt.songid].artist }</div>
</a>
</td>
<td className="center">
<div>
<a href={Link.get('individual_score', attempt.songid, this.convertChart(attempt.chart))}>{
this.convertChart(attempt.chart)
}</a>
</div>
</td>
<td>{ this.renderScore(attempt) }</td>
</tr>
);
}.bind(this))}
</tbody>
<tfoot>
<tr>
<td colSpan={6}>
{ this.state.offset > 0 ?
<Prev onClick={function(event) {
var page = this.state.offset - this.state.limit;
if (page < 0) { page = 0; }
this.setState({offset: page});
}.bind(this)}/> : null
}
{ (this.state.offset + this.state.limit) < this.state.attempts.length ?
<Next style={ {float: 'right'} } onClick={function(event) {
var page = this.state.offset + this.state.limit;
if (page >= this.state.attempts.length) { return }
this.setState({offset: page});
}.bind(this)}/> :
this.state.loading ?
<span className="loading" style={ {float: 'right' } }>
<img
className="loading"
src={Link.get('static', window.assets + 'loading-16.gif')}
/> loading more scores...
</span> : null
}
</td>
</tr>
</tfoot>
</table>
</div>
);
},
});
ReactDOM.render(
React.createElement(network_scores, null),
document.getElementById('content')
);

2
bemani/utils/read.py
View File

@ -6186,7 +6186,7 @@ class ImportDanceEvolution(ImportBase):
# Import it
self.start_batch()
for chart_id in [0, 1, 2, 3, 4, 5]:
for chart_id in [0, 1, 2, 3, 4]:
# First, try to find in the DB from another version
old_id = self.get_music_id_for_song(song["id"], chart_id)
if self.no_combine or old_id is None: