Add recurring CLI scrape automation

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-09-29 12:39:05 -05:00 · 2026-05-24 17:04:07 -05:00
parent 48b52041a5
commit 43f5fa3b71
16 changed files with 1299 additions and 0 deletions
--- a/scripts/tests/container-smoke.sh
+++ b/scripts/tests/container-smoke.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
+TMP_ENV=$(mktemp "${TMPDIR:-/tmp}/dce-container-smoke.XXXXXX.env")
+
+cleanup() {
+  rm -f "$TMP_ENV"
+}
+trap cleanup EXIT
+
+cat >"$TMP_ENV" <<EOF
+DISCORD_TOKEN=dummy
+DCE_UID=$(id -u)
+DCE_GID=$(id -g)
+TZ=UTC
+EOF
+
+cd "$REPO_ROOT"
+docker compose --env-file "$TMP_ENV" build
+docker compose --env-file "$TMP_ENV" run --rm discord-scraper help >/dev/null
+docker compose --env-file "$TMP_ENV" run --rm discord-scraper list-targets >/dev/null
+
+echo "container smoke test passed"
--- a/scripts/tests/run-discord-scrape-smoke.sh
+++ b/scripts/tests/run-discord-scrape-smoke.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
+FIXTURE_DIR="$REPO_ROOT/scripts/tests/test-fixtures"
+TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/dce-run-smoke.XXXXXX")
+ARCHIVE_ROOT="$TMP_DIR/archive"
+CONFIG_PATH="$TMP_DIR/config.json"
+FAKE_CLI="$TMP_DIR/fake-cli.sh"
+
+cleanup() {
+  rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT
+
+cat >"$CONFIG_PATH" <<JSON
+{
+  "archive_root": "$ARCHIVE_ROOT",
+  "defaults": {
+    "include_threads": "all",
+    "include_voice_channels": false
+  },
+  "targets": [
+    {
+      "name": "demo",
+      "kind": "guild",
+      "output_dir": "$ARCHIVE_ROOT/demo",
+      "channel_ids": ["111"],
+      "guild_ids": [],
+      "guild_name_patterns": []
+    }
+  ]
+}
+JSON
+
+cat >"$FAKE_CLI" <<'EOF'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+mode=${FAKE_DCE_MODE:?}
+fixture_dir=${FAKE_DCE_FIXTURE_DIR:?}
+subcommand=${1:?}
+shift || true
+
+case "$subcommand" in
+  export)
+    output=""
+    while (($#)); do
+      case "$1" in
+        --output)
+          output=$2
+          shift 2
+          ;;
+        --channel|--format|--after)
+          shift 2
+          ;;
+        *)
+          shift
+          ;;
+      esac
+    done
+
+    case "$mode" in
+      initial) cp "$fixture_dir/append-existing.json" "$output" ;;
+      append) cp "$fixture_dir/append-incremental.json" "$output" ;;
+      wrong-channel) cp "$fixture_dir/wrong-channel.json" "$output" ;;
+      *) echo "unexpected mode: $mode" >&2; exit 1 ;;
+    esac
+    ;;
+  *)
+    echo "unexpected subcommand: $subcommand" >&2
+    exit 1
+    ;;
+esac
+EOF
+chmod +x "$FAKE_CLI"
+
+run_wrapper() {
+  DISCORD_TOKEN=dummy \
+  DCE_CLI_BIN="$FAKE_CLI" \
+  DCE_PRIMARY_CONFIG="$CONFIG_PATH" \
+  DCE_FALLBACK_CONFIG="$CONFIG_PATH" \
+  FAKE_DCE_FIXTURE_DIR="$FIXTURE_DIR" \
+  FAKE_DCE_MODE="$1" \
+  "$REPO_ROOT/scripts/run-discord-scrape.sh" scrape --target demo
+}
+
+run_wrapper initial
+
+DEST="$ARCHIVE_ROOT/demo/channels/111.json"
+[[ -f "$DEST" ]] || { echo "expected destination archive missing" >&2; exit 1; }
+[[ "$(jq -r '.messages | length' "$DEST")" == "2" ]] || { echo "expected initial message count of 2" >&2; exit 1; }
+
+run_wrapper append
+[[ "$(jq -r '.messages | length' "$DEST")" == "3" ]] || { echo "expected appended message count of 3" >&2; exit 1; }
+[[ "$(jq -r '.messages[-1].id' "$DEST")" == "3" ]] || { echo "expected last message id 3 after append" >&2; exit 1; }
+
+before_checksum=$(sha256sum "$DEST" | awk '{print $1}')
+if run_wrapper wrong-channel; then
+  echo "wrong-channel fixture should have failed" >&2
+  exit 1
+fi
+after_checksum=$(sha256sum "$DEST" | awk '{print $1}')
+[[ "$before_checksum" == "$after_checksum" ]] || { echo "destination archive changed after failed wrong-channel run" >&2; exit 1; }
+
+echo "run-discord-scrape smoke test passed"
--- a/scripts/tests/setup-cron-smoke.sh
+++ b/scripts/tests/setup-cron-smoke.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
+TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/dce-cron-smoke.XXXXXX")
+ARCHIVE_ROOT="$TMP_DIR/archive"
+CONFIG_PATH="$TMP_DIR/config.json"
+ENV_PATH="$TMP_DIR/scrape.env"
+CRONTAB_FILE="$TMP_DIR/crontab.txt"
+DOCKER_LOG="$TMP_DIR/docker.log"
+FAKE_DOCKER="$TMP_DIR/docker"
+FAKE_CRONTAB="$TMP_DIR/crontab"
+
+cleanup() {
+  rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT
+
+cat >"$CONFIG_PATH" <<JSON
+{
+  "archive_root": "$ARCHIVE_ROOT",
+  "defaults": {
+    "include_threads": "all",
+    "include_voice_channels": false
+  },
+  "targets": [
+    {
+      "name": "demo",
+      "kind": "guild",
+      "output_dir": "$ARCHIVE_ROOT/demo",
+      "channel_ids": ["111"],
+      "guild_ids": [],
+      "guild_name_patterns": []
+    }
+  ]
+}
+JSON
+
+cat >"$ENV_PATH" <<EOF
+DISCORD_TOKEN=dummy
+DCE_UID=1000
+DCE_GID=1000
+TZ=UTC
+EOF
+
+cat >"$FAKE_DOCKER" <<EOF
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf '%s\n' "\$*" >>"$DOCKER_LOG"
+exit 0
+EOF
+chmod +x "$FAKE_DOCKER"
+
+cat >"$FAKE_CRONTAB" <<EOF
+#!/usr/bin/env bash
+set -Eeuo pipefail
+file="$CRONTAB_FILE"
+if [[ "\${1:-}" == "-l" ]]; then
+  [[ -f "\$file" ]] || exit 1
+  cat "\$file"
+elif [[ "\${1:-}" == "-" ]]; then
+  cat >"\$file"
+else
+  echo "unexpected crontab args: \$*" >&2
+  exit 1
+fi
+EOF
+chmod +x "$FAKE_CRONTAB"
+
+printf 'MAILTO=test@example.com\n' >"$CRONTAB_FILE"
+
+run_setup() {
+  DCE_CONFIG_FILE="$CONFIG_PATH" \
+  DCE_ENV_FILE="$ENV_PATH" \
+  DCE_CRONTAB_BIN="$FAKE_CRONTAB" \
+  DCE_DOCKER_BIN="$FAKE_DOCKER" \
+  DCE_JQ_BIN="$(command -v jq)" \
+  DCE_REPO_ROOT="$REPO_ROOT" \
+  DCE_LOG_FILE="$TMP_DIR/logs/discord-scrape.log" \
+  "$REPO_ROOT/scripts/setup-cron.sh" --target demo "$@"
+}
+
+run_setup
+grep -q '^MAILTO=test@example.com$' "$CRONTAB_FILE" || { echo "expected unrelated crontab line to remain" >&2; exit 1; }
+[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after install" >&2; exit 1; }
+grep -q 'compose --env-file' "$DOCKER_LOG" || { echo "expected docker preflight to run during install" >&2; exit 1; }
+
+run_setup
+[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "expected exactly one managed cron block after reinstall" >&2; exit 1; }
+
+preview_output=$(run_setup --dry-run)
+grep -q '^# BEGIN discord-scrape$' <<<"$preview_output" || { echo "expected dry-run preview to contain managed block" >&2; exit 1; }
+[[ "$(grep -c '^# BEGIN discord-scrape$' "$CRONTAB_FILE")" == "1" ]] || { echo "dry-run should not alter crontab state" >&2; exit 1; }
+
+run_setup --remove
+grep -q '^MAILTO=test@example.com$' "$CRONTAB_FILE" || { echo "expected unrelated crontab line to remain after remove" >&2; exit 1; }
+! grep -q '^# BEGIN discord-scrape$' "$CRONTAB_FILE" || { echo "expected managed cron block to be removed" >&2; exit 1; }
+
+echo "setup-cron smoke test passed"
--- a/scripts/tests/test-fixtures/append-existing.json
+++ b/scripts/tests/test-fixtures/append-existing.json
@@ -0,0 +1,22 @@
+{
+  "channel": {
+    "id": "111"
+  },
+  "messages": [
+    {
+      "id": "1",
+      "timestamp": "2026-01-01T00:00:00Z",
+      "content": "first"
+    },
+    {
+      "id": "2",
+      "timestamp": "2026-01-02T00:00:00Z",
+      "content": "second"
+    }
+  ],
+  "dateRange": {
+    "after": null,
+    "before": null
+  },
+  "exportedAt": "2026-01-02T00:00:00Z"
+}
--- a/scripts/tests/test-fixtures/append-incremental.json
+++ b/scripts/tests/test-fixtures/append-incremental.json
@@ -0,0 +1,22 @@
+{
+  "channel": {
+    "id": "111"
+  },
+  "messages": [
+    {
+      "id": "2",
+      "timestamp": "2026-01-02T00:00:00Z",
+      "content": "second"
+    },
+    {
+      "id": "3",
+      "timestamp": "2026-01-03T00:00:00Z",
+      "content": "third"
+    }
+  ],
+  "dateRange": {
+    "after": "2026-01-02T00:00:00Z",
+    "before": null
+  },
+  "exportedAt": "2026-01-03T00:00:00Z"
+}
--- a/scripts/tests/test-fixtures/wrong-channel.json
+++ b/scripts/tests/test-fixtures/wrong-channel.json
@@ -0,0 +1,17 @@
+{
+  "channel": {
+    "id": "999"
+  },
+  "messages": [
+    {
+      "id": "4",
+      "timestamp": "2026-01-04T00:00:00Z",
+      "content": "wrong"
+    }
+  ],
+  "dateRange": {
+    "after": "2026-01-03T00:00:00Z",
+    "before": null
+  },
+  "exportedAt": "2026-01-04T00:00:00Z"
+}