feat(scrape): add verify-operator-ready host checks

One command validates compose, auth, config, and seeded archives before
bootstrap or cron. Includes offline smoke test (14 smokes total).
This commit is contained in:
Boden
2026-05-29 14:16:10 -05:00
parent 927d5e9607
commit 00bcbc5b21
7 changed files with 219 additions and 2 deletions

View File

@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -Eeuo pipefail
REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
VERIFY="$REPO_ROOT/scripts/verify-operator-ready.sh"
TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/dce-op-ready-smoke.XXXXXX")
ARCHIVE_ROOT="$TMP_DIR/archive"
CONFIG_PATH="$TMP_DIR/config.json"
ENV_PATH="$TMP_DIR/scrape.env"
FAKE_DOCKER="$TMP_DIR/docker"
PATH_BACKUP="$PATH"
cleanup() {
export PATH="$PATH_BACKUP"
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
mkdir -p "$ARCHIVE_ROOT/demo"
printf '{"messages":[{"id":"1"}],"channel":{"id":"111111111111111111"}}\n' \
>"$ARCHIVE_ROOT/demo/Guild - general [111111111111111111].json"
cat >"$CONFIG_PATH" <<JSON
{
"archive_root": "$ARCHIVE_ROOT",
"targets": [
{
"name": "demo",
"kind": "guild",
"output_dir": "$ARCHIVE_ROOT/demo",
"enabled": true
}
]
}
JSON
printf 'DISCORD_TOKEN=dummy\n' >"$ENV_PATH"
cat >"$FAKE_DOCKER" <<'EOF'
#!/usr/bin/env bash
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then
exit 0
fi
exit 1
EOF
chmod +x "$FAKE_DOCKER"
export PATH="$TMP_DIR:$PATH_BACKUP"
DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" DCE_ENV_FILE="$ENV_PATH" \
"$VERIFY" --config "$CONFIG_PATH"
if DCE_REPO_ROOT="$REPO_ROOT" DCE_CONFIG_FILE="$CONFIG_PATH" DCE_ENV_FILE="$ENV_PATH" \
"$VERIFY" --config "$CONFIG_PATH" --preflight demo 2>/dev/null; then
printf 'ERROR: preflight should fail without real container/token\n' >&2
exit 1
fi
printf 'verify-operator-ready-smoke: ok\n'

119
scripts/verify-operator-ready.sh Executable file
View File

@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
REPO_ROOT="${DCE_REPO_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd -P)}"
CONFIG_PATH="${DCE_CONFIG_FILE:-$REPO_ROOT/config/scrape-targets.json}"
ENV_FILE="${DCE_ENV_FILE:-$REPO_ROOT/scrape.env}"
HOST_RUNNER="$REPO_ROOT/scripts/run-discord-scrape-host.sh"
VERIFY_ARCHIVES="$REPO_ROOT/scripts/verify-documents-archives.sh"
DISCOVER="$REPO_ROOT/scripts/discover-discord-token.sh"
PREFLIGHT_TARGET=""
usage() {
cat <<EOF
Usage:
$(basename "$0") [--config PATH] [--preflight TARGET]
Check host prerequisites for recurring scrape:
jq, container compose, Discord auth, valid config, seeded archives.
With --preflight TARGET, also run Discord preflight for one target.
EOF
}
die() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing."
}
resolve_compose() {
if [[ -n "${DCE_COMPOSE_BIN:-}" ]]; then
printf 'compose: %s\n' "$DCE_COMPOSE_BIN"
return 0
fi
if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
printf 'compose: docker compose\n'
return 0
fi
if command -v docker-compose >/dev/null 2>&1; then
printf 'compose: docker-compose\n'
return 0
fi
if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then
printf 'compose: podman compose\n'
return 0
fi
die "Install Docker or Podman with compose support."
}
check_auth() {
if [[ -f "$ENV_FILE" ]] && grep -qE '^[[:space:]]*DISCORD_TOKEN=' "$ENV_FILE"; then
printf 'auth: scrape.env has DISCORD_TOKEN\n'
return 0
fi
if [[ -n "${DISCORD_TOKEN:-}" ]]; then
printf 'auth: DISCORD_TOKEN exported in environment\n'
return 0
fi
if [[ -x "$DISCOVER" ]]; then
local token
token=$("$DISCOVER" 2>/dev/null || true)
if [[ -n "$token" ]]; then
printf 'auth: token discoverable (GUI or config paths)\n'
return 0
fi
fi
die "No Discord token: set scrape.env, export DISCORD_TOKEN, or sync from GUI."
}
main() {
while (($#)); do
case "$1" in
--config)
[[ $# -ge 2 ]] || die "Missing value for --config."
CONFIG_PATH=$2
shift 2
;;
--preflight)
[[ $# -ge 2 ]] || die "Missing value for --preflight."
PREFLIGHT_TARGET=$2
shift 2
;;
--help|-h)
usage
exit 0
;;
*)
die "Unknown option: $1"
;;
esac
done
require_command jq
[[ -f "$CONFIG_PATH" ]] || die "Missing config: $CONFIG_PATH"
jq empty "$CONFIG_PATH" >/dev/null 2>&1 || die "Invalid JSON config: $CONFIG_PATH"
printf 'Operator readiness checks\n'
printf '=========================\n'
resolve_compose
check_auth
printf 'config: %s\n\n' "$CONFIG_PATH"
DCE_PRIMARY_CONFIG="$CONFIG_PATH" "$VERIFY_ARCHIVES" --config "$CONFIG_PATH"
if [[ -n "$PREFLIGHT_TARGET" ]]; then
printf '\nRunning preflight for target %s...\n' "$PREFLIGHT_TARGET"
"$HOST_RUNNER" preflight --config /config/scrape-targets.json --target "$PREFLIGHT_TARGET"
fi
printf '\nOperator ready. Next:\n'
printf ' ./scripts/run-documents-scrape.sh\n'
printf ' ./scripts/setup-cron.sh --dry-run\n'
}
main "$@"