Create verbose mode for all scripts and reduce default output

Man, dealing with passing arguments through multiple programs correctly is annoying. Tip: don't use eval, use "$@" instead.

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
         run: ln -s .github/compose.ci.yml compose.override.yml
 
       - name: Run the setup script
-        run: ./setup.sh --server-ip 127.0.0.1 --force
+        run: ./setup.sh --server-ip 127.0.0.1 --force --verbose
 
       - name: Test starting the servers
         run: |

scripts/backup.sh

@@ -7,6 +7,14 @@ this before doing something risky with the datbases to prevent data loss."
 add_positional_argument "backup-name" "backup_name" "Name of the backup directory, defaults to the current date and time" false
 parse_arguments "$@"
 
+if [[ ! -f "$git_base_dir/environment/minio.local.env" ]]; then
+    print_error "Missing environment file minio.local.env. Did you run setup-environment.sh?"
+    exit 1
+fi
+source "$git_base_dir/environment/minio.env"
+source "$git_base_dir/environment/minio.local.env"
+source "$git_base_dir/environment/postgres.env"
+
 if [[ -z "$backup_name" ]]; then
     backup_name="backup_$(date +%Y-%m-%dT%H.%M.%S)"
 fi
@@ -22,27 +30,29 @@ print_info "Backing up to $backup_dir"
 docker compose up -d mitmproxy-pretendo mongodb postgres minio redis
 
 print_info "Backing up MongoDB..."
-docker compose exec mongodb rm -rf /tmp/backup
-docker compose exec mongodb mongodump -o /tmp/backup --quiet
-docker compose cp mongodb:/tmp/backup "$backup_dir/mongodb"
-docker compose exec mongodb rm -rf /tmp/backup
+run_verbose docker compose exec mongodb rm -rf /tmp/backup
+# mongodump uses stderr for output
+[[ -z "$show_verbose" ]] && mongodump_quiet=true
+run_verbose docker compose exec mongodb mongodump -o /tmp/backup "${mongodump_quiet:+--quiet}"
+run_verbose_no_errors docker compose cp mongodb:/tmp/backup "$backup_dir/mongodb"
+run_verbose docker compose exec mongodb rm -rf /tmp/backup
 
 print_info "Backing up Postgres..."
-docker compose exec postgres sh -c 'pg_dumpall -U "$POSTGRES_USER" --clean' >"$backup_dir/postgres.sql"
+docker compose exec postgres pg_dumpall -U "$POSTGRES_USER" --clean >"$backup_dir/postgres.sql"
 
 print_info "Backing up MinIO..."
-docker compose exec minio sh -c 'mc alias set minio http://minio.pretendo.cc "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"'
-docker compose exec minio rm -rf /tmp/backup
-docker compose exec minio mkdir -p /tmp/backup
-docker compose exec minio mc mirror minio/ /tmp/backup
-docker compose cp minio:/tmp/backup "$backup_dir/minio"
-docker compose exec minio rm -rf /tmp/backup
+run_verbose docker compose exec minio mc alias set minio http://minio.pretendo.cc "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+run_verbose docker compose exec minio rm -rf /tmp/backup
+run_verbose docker compose exec minio mkdir -p /tmp/backup
+run_verbose docker compose exec minio mc mirror minio/ /tmp/backup
+run_verbose_no_errors docker compose cp minio:/tmp/backup "$backup_dir/minio"
+run_verbose docker compose exec minio rm -rf /tmp/backup
 
 print_info "Backing up Redis..."
-docker compose exec redis redis-cli save
-docker compose cp redis:/data/dump.rdb "$backup_dir/redis.rdb"
+run_verbose docker compose exec redis redis-cli save
+run_verbose_no_errors docker compose cp redis:/data/dump.rdb "$backup_dir/redis.rdb"
 
 print_info "Backing up Mitmproxy..."
-docker compose cp mitmproxy-pretendo:/home/mitmproxy/.mitmproxy "$backup_dir/mitmproxy"
+run_verbose_no_errors docker compose cp mitmproxy-pretendo:/home/mitmproxy/.mitmproxy "$backup_dir/mitmproxy"
 
 print_success "Backup completed successfully."

scripts/compile-custom-inkay.sh

@@ -21,11 +21,11 @@ cd "$git_base_dir/repos/Inkay"
 
 if [[ -z "$should_reset" ]]; then
     docker compose up -d mitmproxy-pretendo
-    run_command_until_success "docker compose exec mitmproxy-pretendo ls /home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" \
-        "Waiting for mitmproxy to generate a certificate..." 4
+    run_command_until_success "Waiting for mitmproxy to generate a certificate..." 5 \
+        docker compose exec mitmproxy-pretendo ls /home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem
 
     # Get the current certificate
-    docker compose cp mitmproxy-pretendo:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./data/ca.pem
+    run_verbose_no_errors docker compose cp mitmproxy-pretendo:/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem ./data/ca.pem
 else
     git restore ./data/ca.pem
     print_info "Reset Inkay CA certificate."

scripts/create-juxt-community.sh

@@ -19,10 +19,10 @@ title_ids=$(echo "$title_ids" | tr -dc "a-zA-Z0-9,")
 docker compose up -d juxtaposition-ui
 
 if [[ -n "$icon_path" ]]; then
-    docker compose cp "$icon_path" juxtaposition-ui:/tmp/icon
+    run_verbose_no_errors docker compose cp "$icon_path" juxtaposition-ui:/tmp/icon
 fi
 if [[ -n "$banner_path" ]]; then
-    docker compose cp "$banner_path" juxtaposition-ui:/tmp/banner
+    run_verbose_no_errors docker compose cp "$banner_path" juxtaposition-ui:/tmp/banner
 fi
 
 docker compose exec juxtaposition-ui node -e "$create_community_script" "$name" "$description" "$title_ids" "${icon_path:+/tmp/icon}" "${banner_path:+/tmp/banner}"

scripts/full-reset.sh

@@ -27,5 +27,7 @@ docker compose down --volumes
 rm -f ./environment/*.local.env
 
 print_success "Reset complete. Run setup.sh to set up the environment again."
-print_info "To restore the backup and undo the reset, use scripts/restore.sh with the backup directory starting with
+if [[ -z "$no_backup" ]]; then
+    print_info "To restore the backup and undo the reset, use scripts/restore.sh with the backup directory starting with
 \"pre_reset\" after running setup.sh."
+fi

scripts/internal/firstrun-minio-container.sh

@@ -4,12 +4,18 @@
 source "$(dirname "$(realpath "$0")")/framework.sh"
 parse_arguments "$@"
 
+if [[ ! -f "$git_base_dir/environment/minio.local.env" ]]; then
+    print_error "Missing environment file minio.local.env. Did you run setup-environment.sh?"
+    exit 1
+fi
+source "$git_base_dir/environment/minio.env"
+source "$git_base_dir/environment/minio.local.env"
+
 minio_init_script=$(cat "$git_base_dir/scripts/run-in-container/minio-init.sh")
 
 docker compose up -d minio
 
-# sh -c is needed to expand environment variables inside the container
-run_command_until_success "docker compose exec minio sh -c 'mc alias set minio http://minio.pretendo.cc \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\"'" \
-    "Waiting for MinIO to be ready..." 2
+run_command_until_success "Waiting for MinIO to be ready..." 5 \
+    docker compose exec minio mc alias set minio http://minio.pretendo.cc "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
 
 docker compose exec minio sh -c "$minio_init_script"

scripts/internal/firstrun-mongodb-container.sh

@@ -8,12 +8,11 @@ mongodb_init_script=$(cat "$git_base_dir/scripts/run-in-container/mongodb-init.j
 
 docker compose up -d mongodb
 
-# This won't work in /docker-entrypoint-initdb.d/ because MongoDB is in a
-# special init state where it will refuse to resolve anything but localhost.
-# This needs to be run after it initializes.
+# This won't work in /docker-entrypoint-initdb.d/ because MongoDB is in a special init state where it will refuse to
+# resolve anything but localhost. This needs to be run after it initializes.
 # https://github.com/docker-library/mongo/issues/339
 
-run_command_until_success "docker compose exec mongodb mongosh --eval 'db.adminCommand(\"ping\")'" \
-    "Waiting for MongoDB to be ready..."
+run_command_until_success "Waiting for MongoDB to be ready..." 10 \
+    docker compose exec mongodb mongosh --eval "db.adminCommand('ping')"
 
 docker compose exec mongodb mongosh --eval "$mongodb_init_script"

scripts/internal/framework.sh

@@ -83,22 +83,41 @@ print_success() {
     echo "${term_bold}${term_green}${*}${term_reset}"
 }
 
-# Run a command every second silently until it succeeds, or show output if the max number of retries is reached.
+# Run a command and only show output if the verbose option is set, show errors
+run_verbose() {
+    if [[ -n "$show_verbose" ]]; then
+        "$@"
+    else
+        "$@" >/dev/null
+    fi
+}
+
+# Run a command and only show output if the verbose option is set, hide errors
+run_verbose_no_errors() {
+    if [[ -n "$show_verbose" ]]; then
+        "$@"
+    else
+        "$@" >/dev/null 2>&1
+    fi
+}
+
+# Run a command every 2 seconds silently until it succeeds, or show output if the max number of retries is reached.
 #
-# Usage: run_command_until_success command wait_text [max_attempts]
-# Example: run_command_until_success "docker compose exec mongodb mongosh --eval 'db.adminCommand(\"ping\")'" "Waiting
-# for MongoDB to be ready..." 10
+# Usage: run_command_until_success wait_text max_attempts command...
+# Example: run_command_until_success "Waiting for command..." 5 command arg1 arg2
 run_command_until_success() {
-    local command="${1:?${FUNCNAME[0]}: Command is required}"
-    local wait_text="${2:?${FUNCNAME[0]}: Wait text is required}"
-    local max_attempts="${3:-10}"
+    local wait_text="${1:?${FUNCNAME[0]}: Wait text is required}"
+    local max_attempts="${2:?${FUNCNAME[0]}: Max attempts is required}"
+    shift
+    shift
     local count=0
 
-    while ! eval $command >/dev/null 2>&1; do
+    while ! run_verbose_no_errors "$@"; do
         count=$((count + 1))
-        if [ $count -ge $max_attempts ]; then
+        if [[ $count -ge "$max_attempts" ]]; then
             print_error "Max attempts reached. Showing error info..."
-            eval $command
+            "$@"
+            exit 1
         fi
         print_info "$wait_text"
         sleep 2
@@ -247,6 +266,9 @@ parse_arguments() {
         declare -g "$variable="
     done
 
+    # Keep verbosity if it was set in a previous script
+    show_verbose="${SHOW_VERBOSE:-}"
+
     # Split combined short options into individual options
     local split_args=()
     local arg
@@ -308,6 +330,9 @@ parse_arguments() {
             exit 1
         fi
     done
+
+    # Export verbosity for the next script
+    export SHOW_VERBOSE="${show_verbose:-}"
 }
 
 # Shows the auto-generated help text based on the configured options and positional arguments.
@@ -336,3 +361,4 @@ show_help() {
 }
 
 add_option "-h --help" "show_help" "Displays this help message"
+add_option "-v --verbose" "show_verbose" "Enables verbose output"

scripts/internal/update-postgres-password.sh

@@ -13,10 +13,10 @@ source "$git_base_dir/environment/postgres.local.env"
 
 docker compose up -d postgres
 
-run_command_until_success "docker compose exec postgres psql -v ON_ERROR_STOP=1 -U '$POSTGRES_USER' -c '\l'" \
-    "Waiting for Postgres to be ready..."
+run_command_until_success "Waiting for Postgres to be ready..." 5 \
+    docker compose exec postgres psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -l
 
-# During the first run, this sometimes fails because the entrypoint script
-# restarts the server after running the initdb scripts
-run_command_until_success "docker compose exec postgres psql -U '$POSTGRES_USER' -c \"ALTER USER $POSTGRES_USER PASSWORD '$POSTGRES_PASSWORD';\"" \
-    "Failed to change Postgres password, retrying..." 5
+# During the first run, this sometimes fails because the entrypoint script restarts the server after running the initdb
+# scripts
+run_command_until_success "Failed to change Postgres password, retrying..." 5 \
+    docker compose exec postgres psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "ALTER USER $POSTGRES_USER PASSWORD '$POSTGRES_PASSWORD';"

scripts/restore.sh

@@ -10,6 +10,14 @@ add_positional_argument "backup-directory" "backup_dir" "The backup directory to
 add_option "-f --force" "force" "Skips the restore confirmation prompt"
 parse_arguments "$@"
 
+if [[ ! -f "$git_base_dir/environment/minio.local.env" ]]; then
+    print_error "Missing environment file minio.local.env. Did you run setup-environment.sh?"
+    exit 1
+fi
+source "$git_base_dir/environment/minio.env"
+source "$git_base_dir/environment/minio.local.env"
+source "$git_base_dir/environment/postgres.env"
+
 if [[ ! -d "$backup_dir" ]]; then
     print_error "Backup directory $backup_dir does not exist."
     exit 1
@@ -31,31 +39,32 @@ docker compose down
 docker compose up -d mitmproxy-pretendo mongodb postgres minio redis
 
 print_info "Restoring MongoDB..."
-docker compose exec mongodb rm -rf /tmp/backup
-docker compose cp "$backup_dir/mongodb" mongodb:/tmp/backup
-docker compose exec mongodb mongorestore /tmp/backup --drop --quiet
-docker compose exec mongodb rm -rf /tmp/backup
+run_verbose docker compose exec mongodb rm -rf /tmp/backup
+run_verbose_no_errors docker compose cp "$backup_dir/mongodb" mongodb:/tmp/backup
+# mongodump uses stderr for output
+[[ -z "$show_verbose" ]] && mongodump_quiet=true
+run_verbose docker compose exec mongodb mongorestore /tmp/backup --drop "${mongodump_quiet:+--quiet}"
+run_verbose docker compose exec mongodb rm -rf /tmp/backup
 
 print_info "Restoring Postgres..."
 # According to the pg_dumpall documentation, dropping and creating the superuser role is expected to cause an error
-print_info "Note: the errors \"current user cannot be dropped\" and \"role ... already exists\" are expected and can be safely ignored."
-docker compose exec -T postgres sh -c 'psql -U "$POSTGRES_USER" -d postgres' <"$backup_dir/postgres.sql" >/dev/null
+run_verbose_no_errors docker compose exec -T postgres psql -U "$POSTGRES_USER" -d postgres <"$backup_dir/postgres.sql"
 
 print_info "Restoring MinIO..."
-docker compose exec minio /bin/sh -c 'mc alias set minio http://minio.pretendo.cc "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"'
-docker compose exec minio rm -rf /tmp/backup
-docker compose cp "$backup_dir/minio" minio:/tmp/backup
-docker compose exec minio mc mirror /tmp/backup minio/ --overwrite --remove
-docker compose exec minio rm -rf /tmp/backup
+run_verbose docker compose exec minio mc alias set minio http://minio.pretendo.cc "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+run_verbose docker compose exec minio rm -rf /tmp/backup
+run_verbose_no_errors docker compose cp "$backup_dir/minio" minio:/tmp/backup
+run_verbose docker compose exec minio mc mirror /tmp/backup minio/ --overwrite --remove
+run_verbose docker compose exec minio rm -rf /tmp/backup
 
 print_info "Restoring Redis..."
 # Redis cannot be running when restoring a dump or it will overwrite the restored dump when it exits
-docker compose stop redis
-docker compose cp "$backup_dir/redis.rdb" redis:/data/dump.rdb
-docker compose start redis
+run_verbose_no_errors docker compose stop redis
+run_verbose_no_errors docker compose cp "$backup_dir/redis.rdb" redis:/data/dump.rdb
+run_verbose_no_errors docker compose start redis
 
 print_info "Restoring Mitmproxy..."
-docker compose cp "$backup_dir/mitmproxy" mitmproxy-pretendo:/home/mitmproxy/.mitmproxy
+run_verbose_no_errors docker compose cp "$backup_dir/mitmproxy" mitmproxy-pretendo:/home/mitmproxy/.mitmproxy
 
 # The restored backup might be using different secrets than what are currently in the .env files
 "$git_base_dir/scripts/setup-environment.sh" --force

scripts/run-in-container/make-pnid-dev.js

@@ -17,7 +17,7 @@ async function runAsync() {
     if (pnid) {
         await updatePnidAccessLevel(pnid, 3, "dev");
     } else {
-        console.log(`No PNID found for username ${process.argv[1]}.`);
+        console.error(`No PNID found for username ${process.argv[1]}.`);
         await mongoose.connection.close();
         process.exit(2);
     }

scripts/setup-submodule-patches.sh

@@ -9,10 +9,10 @@ be applied first with --reject, and if that fails, a 3-way merge will be attempt
 parse_arguments "$@"
 
 print_info "Resetting all submodules..."
-git submodule sync >/dev/null
-git submodule foreach "git reset --hard" >/dev/null
-git submodule foreach "git clean -fd" >/dev/null
-git submodule update --init --checkout >/dev/null
+run_verbose git submodule sync
+run_verbose git submodule foreach "git reset --hard"
+run_verbose git submodule foreach "git clean -fd"
+run_verbose git submodule update --init --checkout
 if [[ -n "$update_remote" ]]; then
     print_info "Updating submodules from their remotes..."
     git submodule update --remote
@@ -41,6 +41,7 @@ for dir in "$git_base_dir/patches/"*; do
                     error_count=$((error_count + 1))
                 fi
             else
+                run_verbose echo "Applying patch $patch"
                 git apply "$patch"
             fi
             patch_count=$((patch_count + 1))

setup.sh

@@ -5,19 +5,19 @@ set -euo pipefail
 check_prerequisites() {
     prerequisites_failed=
     prerequisites_warning=
-    if ! docker version >/dev/null; then
+    if ! run_verbose docker version; then
         print_error "Docker is not installed. Please install it: https://docs.docker.com/get-docker/"
         print_info "If you see a \"Permission denied while trying to connect to the Docker daemon\" error, you need to \
 add your user to the docker group: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user."
         prerequisites_failed=true
     fi
-    if ! docker compose version >/dev/null; then
+    if ! run_verbose docker compose version; then
         print_error "Docker Compose is not installed. Please install it: https://docs.docker.com/compose/install/"
         prerequisites_failed=true
     fi
     # The tnftp "enhanced ftp client" has the -u option for direct uploads,
     # unlike the netkit-ftp "classical ftp client"
-    if ! command -v tnftp >/dev/null; then
+    if ! run_verbose command -v tnftp; then
         print_warning "tnftp is not installed. You will not be able to upload files to your consoles automatically."
         prerequisites_warning=true
     fi
@@ -88,8 +88,6 @@ if ! git --version >/dev/null; then
     exit 1
 fi
 
-git config --local submodule.recurse true
-
 # shellcheck source=./scripts/internal/framework.sh
 source "$(dirname "$(realpath "$0")")/scripts/internal/framework.sh"
 set_description "This is the main setup script for your self-hosted Pretendo Network server. By default, it will prompt \
@@ -105,6 +103,8 @@ parse_arguments "$@"
 
 print_title "Unofficial Pretendo Network server setup script started"
 
+git config --local submodule.recurse true
+
 print_stage "Checking prerequisites."
 check_prerequisites
 

update.sh

@@ -6,7 +6,7 @@ set_description "This updates the Pretendo environment to the latest version."
 parse_arguments "$@"
 
 # Without resetting, Git may have merge conflicts in the submodules because of the applied patches
-git submodule foreach "git reset --hard" >/dev/null
+run_verbose git submodule foreach git reset --hard
 git pull
 
 exec "$git_base_dir/setup.sh"