Leahnaya/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2026-09-30 21:20:09 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(scrape): auth discovery, skip forbidden channels, mount host script

Browse Source 
Discover Discord tokens from env, token files, GUI Settings.dat, and desktop
leveldb; bind-mount the host scrape script so container preflight uses
partition/--after cursors; skip inaccessible channels without aborting targets;
fix set -e and busybox mktemp for incremental exports under ~/Documents.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Boden
2026-05-28 14:57:16 -05:00
parent 88e864c72a
commit 57d472f8e8
15 changed files with 532 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/tests/container-smoke.sh
View File

@@ -30,6 +30,10 @@ docker compose --env-file "$TMP_ENV" build
docker compose --env-file "$TMP_ENV" run --rm discord-scraper help >/dev/null
docker compose --env-file "$TMP_ENV" run --rm discord-scraper list-targets >/dev/null
docker compose --env-file "$TMP_ENV" run -T --rm --entrypoint /bin/sh discord-scraper -c \
'grep -q -- "--partition 1" /opt/dce-scheduler/run-discord-scrape.sh && ! grep -q "1970-01-01" /opt/dce-scheduler/run-discord-scrape.sh' \
>/dev/null
if docker version 2>&1 | grep -qi podman || docker info 2>&1 | grep -qi podman; then
mkdir -p "$WRITE_TEST_DIR"
docker compose --env-file "$TMP_PODMAN_ENV" run -T --rm --entrypoint /bin/sh discord-scraper -lc "mkdir -p '$WRITE_TEST_DIR/from-container' && rmdir '$WRITE_TEST_DIR/from-container'" >/dev/null

27
scripts/tests/run-discord-scrape-smoke.sh
View File

@@ -110,6 +110,14 @@ cat >"$CONFIG_PATH" <<JSON
"channel_ids": ["111"],
"guild_ids": [],
"guild_name_patterns": []
},
{
"name": "skip-forbidden",
"kind": "guild",
"output_dir": "$ARCHIVE_ROOT/skip-forbidden",
"channel_ids": ["111", "403"],
"guild_ids": [],
"guild_name_patterns": []
}
]
}
@@ -128,6 +136,7 @@ case "$subcommand" in
export)
output=""
after=""
channel=""
while (($#)); do
case "$1" in
--output)
@@ -142,7 +151,11 @@ case "$subcommand" in
fi
shift 2
;;
--channel|--format)
--channel)
channel=$2
shift 2
;;
--format)
shift 2
;;
*)
@@ -151,6 +164,11 @@ case "$subcommand" in
esac
done
if [[ "$channel" == "403" ]]; then
echo "Request to 'channels/403' failed: forbidden." >&2
exit 1
fi
case "$mode" in
initial) cp "$fixture_dir/append-existing.json" "$output" ;;
append) cp "$fixture_dir/append-incremental.json" "$output" ;;
@@ -304,6 +322,13 @@ bootstrap_mapped_dest=$(jq -r '."111"' "$ARCHIVE_ROOT/bootstrap-map/.dce-meta/ch
[[ "$bootstrap_mapped_dest" == "$BOOTSTRAP_DEST" ]] || { echo "expected bootstrap to register existing archive in channel map" >&2; exit 1; }
[[ "$(jq -r '.messages | length' "$BOOTSTRAP_DEST")" == "3" ]] || { echo "expected bootstrap-map archive to append in place" >&2; exit 1; }
mkdir -p "$ARCHIVE_ROOT/skip-forbidden"
cp "$FIXTURE_DIR/append-existing.json" "$ARCHIVE_ROOT/skip-forbidden/$DEFAULT_FILE_NAME"
run_wrapper skip-forbidden append
SKIP_DEST="$ARCHIVE_ROOT/skip-forbidden/$DEFAULT_FILE_NAME"
[[ "$(jq -r '.messages | length' "$SKIP_DEST")" == "3" ]] || { echo "expected skip-forbidden to append accessible channel" >&2; exit 1; }
[[ ! -e "$ARCHIVE_ROOT/skip-forbidden/channels/403.json" ]] || { echo "unexpected fallback file for skipped forbidden channel" >&2; exit 1; }
# shellcheck disable=SC1091
source "$REPO_ROOT/scripts/run-discord-scrape.sh"
SHRINK_EXISTING="$TMP_DIR/shrink-existing.json"