Leahnaya/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-09-29 12:39:05 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(scrape): add salvage-only mode for stale temp exports

Browse Source 
Expose run-discord-scrape.sh salvage and run-documents-scrape.sh
--salvage-only so operators can merge quiescent .dce-temp partials without
re-downloading from Discord after stopping a long or OOM-aborted run.
This commit is contained in:
Copilot
2026-06-03 06:48:39 -05:00
parent 2c01b3a7b9
commit 8468e34e37
6 changed files with 245 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scripts/tests/documents-scrape-smoke.sh
View File

@@ -46,7 +46,7 @@ DCE_REPO_ROOT="$FAKE_REPO" \
ARCHIVE="$TMP_DIR/server"
mkdir -p "$ARCHIVE"
printf '{"messages":[{"id":"1","timestamp":"2020-01-01T00:00:00"}]}\n' >"$ARCHIVE/Guild - general [111111111111111111].json"
printf '{"guild":{"id":"1","name":"Guild"},"channel":{"id":"111111111111111111","name":"general"},"messages":[{"id":"1","timestamp":"2020-01-01T00:00:00"}]}\n' >"$ARCHIVE/Guild - general [111111111111111111].json"
cat >"$TMP_DIR/config.json" <<JSON
{
@@ -112,6 +112,22 @@ grep -q '111111111111111111' "$ARGS_LOG" || {
exit 1
}
cp "$REPO_ROOT/scripts/run-discord-scrape.sh" "$FAKE_REPO/scripts/"
chmod +x "$FAKE_REPO/scripts/run-discord-scrape.sh"
SALVAGE_DOC_LOG="$TMP_DIR/salvage-documents.log"
DCE_MIN_FREE_MB=0 \
DCE_SKIP_SCRAPE_LOCK=1 \
"$REPO_ROOT/scripts/run-documents-scrape.sh" --salvage-only --config "$TMP_DIR/config.json" --target demo >"$SALVAGE_DOC_LOG" 2>&1 || {
echo "salvage-only documents scrape failed" >&2
cat "$SALVAGE_DOC_LOG" >&2
exit 1
}
grep -q 'salvage completed' "$SALVAGE_DOC_LOG" || {
echo "expected --salvage-only to run local salvage" >&2
cat "$SALVAGE_DOC_LOG" >&2
exit 1
}
DCE_MIN_FREE_MB=0 DCE_CONFIG_FILE="$TMP_DIR/config.json" \
"$REPO_ROOT/scripts/verify-operator-ready.sh" --disk-only --config "$TMP_DIR/config.json" \
| grep -q 'disk-only: ok'

33
scripts/tests/run-discord-scrape-smoke.sh
View File

@@ -158,6 +158,14 @@ cat >"$CONFIG_PATH" <<JSON
"channel_ids": ["111"],
"guild_ids": [],
"guild_name_patterns": []
},
{
"name": "salvage-only",
"kind": "guild",
"output_dir": "$ARCHIVE_ROOT/salvage-only",
"channel_ids": ["111"],
"guild_ids": [],
"guild_name_patterns": []
}
]
}
@@ -456,6 +464,31 @@ jq -e '.messages[] | select(.id == "3")' "$SALVAGE_DEST" >/dev/null || { echo "e
[[ ! -d "$ARCHIVE_ROOT/salvage-stale/.dce-temp/export.111.STALE" ]] || { echo "expected stale temp dir cleaned up after salvage" >&2; exit 1; }
grep -q 'SALVAGED' "$SALVAGE_LOG" || { echo "expected SALVAGED line in salvage log" >&2; exit 1; }
mkdir -p "$ARCHIVE_ROOT/salvage-only"
cp "$FIXTURE_DIR/append-existing.json" "$ARCHIVE_ROOT/salvage-only/$DEFAULT_FILE_NAME"
mkdir -p "$ARCHIVE_ROOT/salvage-only/.dce-meta"
printf '{\"111\":\"%s\"}\n' "$ARCHIVE_ROOT/salvage-only/$DEFAULT_FILE_NAME" >"$ARCHIVE_ROOT/salvage-only/.dce-meta/channel-map.json"
mkdir -p "$ARCHIVE_ROOT/salvage-only/.dce-temp/export.111.ONLYSTALE"
cp "$FIXTURE_DIR/salvage-truncated.json" "$ARCHIVE_ROOT/salvage-only/.dce-temp/export.111.ONLYSTALE/export.json"
touch -d '1 hour ago' "$ARCHIVE_ROOT/salvage-only/.dce-temp/export.111.ONLYSTALE/export.json"
SALVAGE_ONLY_LOG="$TMP_DIR/salvage-only.log"
DISCORD_TOKEN=dummy \
DCE_CLI_BIN="$FAKE_CLI" \
DCE_PRIMARY_CONFIG="$CONFIG_PATH" \
DCE_FALLBACK_CONFIG="$CONFIG_PATH" \
FAKE_DCE_FIXTURE_DIR="$FIXTURE_DIR" \
FAKE_DCE_MODE=append \
"$REPO_ROOT/scripts/run-discord-scrape.sh" salvage --config "$CONFIG_PATH" --target salvage-only >"$SALVAGE_ONLY_LOG" 2>&1
grep -q 'Exporting channel' "$SALVAGE_ONLY_LOG" && {
echo "salvage-only should not invoke Discord export" >&2
cat "$SALVAGE_ONLY_LOG" >&2
exit 1
}
grep -q 'salvage completed' "$SALVAGE_ONLY_LOG" || { echo "expected salvage completed in salvage-only log" >&2; exit 1; }
SALVAGE_ONLY_DEST="$ARCHIVE_ROOT/salvage-only/$DEFAULT_FILE_NAME"
SALVAGE_ONLY_COUNT=$(jq -r '.messages | length' "$SALVAGE_ONLY_DEST")
(( SALVAGE_ONLY_COUNT >= 3 )) || { echo "expected salvage-only archive to have at least 3 messages (got $SALVAGE_ONLY_COUNT)" >&2; exit 1; }
# shellcheck disable=SC1091
source "$REPO_ROOT/scripts/run-discord-scrape.sh"
SHRINK_EXISTING="$TMP_DIR/shrink-existing.json"