293 lines
11 KiB
Bash
Executable File
293 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Apply database migrations. Run from the repo root, INSIDE nix-shell (that is
|
|
# where the prisma CLI lives).
|
|
#
|
|
# nix-shell shell.nix --run ./migrateDatabase.sh
|
|
#
|
|
# Works both ways round: it applies pending migrations to an existing database,
|
|
# and builds a complete one from empty. Safe to re-run -- a second run is a
|
|
# no-op. Replaces migrateDatabase.sh.
|
|
#
|
|
# --- why this is a separate step -----------------------------------------
|
|
#
|
|
# The application connects as a least-privilege account (production uses `app`)
|
|
# holding DML rights only -- no CREATE, ALTER or DROP. That is deliberate: the
|
|
# server fetches untrusted URLs for link previews, and a service able to reshape
|
|
# its own tables is a far larger blast radius than that feature warrants.
|
|
#
|
|
# So the app never applies DDL. It verifies the schema at boot
|
|
# (server/helpers/Schema.js) and disables link previews if anything is missing.
|
|
#
|
|
# --- authoring a new migration -------------------------------------------
|
|
#
|
|
# NEVER run `prisma migrate dev`. It can decide the database has drifted and
|
|
# offer to RESET it, which on a box holding a production data restore would
|
|
# destroy real data. It also wants a shadow database. Instead:
|
|
#
|
|
# 1. Edit prisma/schema.prisma
|
|
# 2. NAME=add_whatever
|
|
# DIR="prisma/migrations/$(date +%Y%m%d%H%M%S)_$NAME"
|
|
# mkdir -p "$DIR"
|
|
# prisma migrate diff --from-config-datasource \
|
|
# --to-schema prisma/schema.prisma --script > "$DIR/migration.sql"
|
|
# 3. READ the generated SQL -- this review is what makes the approach safe
|
|
# 4. ./migrateDatabase.sh
|
|
#
|
|
# Note: Prisma emits `COLLATE utf8mb4_unicode_ci`, but this database is
|
|
# `utf8mb4_general_ci` throughout, and `prisma migrate diff` does NOT report the
|
|
# difference. Fix the collation in any generated migration by hand. This script
|
|
# checks for drift at the end.
|
|
#
|
|
# --- credentials ---------------------------------------------------------
|
|
#
|
|
# Same convention as applyLastBackup.sh / backupDatabase.sh: source ~/.env and
|
|
# connect as root with $PROD_DB_PASS. Falls back to passwordless root (the dev
|
|
# case, where shell.nix also exports DATABASE_URL). Override with:
|
|
#
|
|
# DB_ADMIN_USER=admin DB_ADMIN_PASS=secret ./migrateDatabase.sh
|
|
#
|
|
set -euo pipefail
|
|
|
|
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$REPO"
|
|
|
|
DB_NAME="application" # hardcoded in server/config/database.js too
|
|
|
|
# --- preconditions -------------------------------------------------------
|
|
|
|
if ! command -v prisma >/dev/null 2>&1; then
|
|
echo "prisma is not on PATH -- run this inside nix-shell:" >&2
|
|
echo " nix-shell shell.nix --run ./migrateDatabase.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v mariadb >/dev/null 2>&1 && ! command -v mysql >/dev/null 2>&1; then
|
|
echo "no mariadb/mysql client on PATH -- run this inside nix-shell." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if command -v mariadb >/dev/null 2>&1; then MYSQL_BIN="mariadb"; else MYSQL_BIN="mysql"; fi
|
|
|
|
if ! command -v node >/dev/null 2>&1; then
|
|
echo "node is not on PATH -- needed to percent-encode the database password." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -f "$HOME/.env" ]; then
|
|
# shellcheck disable=SC1091
|
|
source "$HOME/.env"
|
|
fi
|
|
|
|
ADMIN_USER="${DB_ADMIN_USER:-root}"
|
|
ADMIN_PASS="${DB_ADMIN_PASS:-${PROD_DB_PASS:-}}"
|
|
|
|
mysql_args=(--user "$ADMIN_USER")
|
|
[ -n "$ADMIN_PASS" ] && mysql_args+=(--password="$ADMIN_PASS")
|
|
|
|
LOCAL_SOCKET="$REPO/.mysql/mysql.sock"
|
|
[ -S "$LOCAL_SOCKET" ] && mysql_args+=(--socket="$LOCAL_SOCKET")
|
|
|
|
# Prisma needs a URL, and it must carry the ADMIN credentials -- never the
|
|
# app account, which holds no DDL.
|
|
DB_ADMIN_HOST="${DB_ADMIN_HOST:-127.0.0.1}"
|
|
DB_ADMIN_PORT="${DB_ADMIN_PORT:-3306}"
|
|
|
|
#
|
|
# Percent-encode the password. Production passwords routinely contain @ : / ? #
|
|
# or %, every one of which silently corrupts a URL -- an unencoded `@` would
|
|
# truncate the host and produce a baffling connection error rather than an
|
|
# obvious one.
|
|
#
|
|
encode_component() {
|
|
node -e 'process.stdout.write(encodeURIComponent(process.argv[1] || ""))' "$1"
|
|
}
|
|
|
|
ADMIN_USER_ENC="$(encode_component "$ADMIN_USER")"
|
|
|
|
if [ -n "$ADMIN_PASS" ]; then
|
|
ADMIN_PASS_ENC="$(encode_component "$ADMIN_PASS")"
|
|
export DATABASE_URL="mysql://${ADMIN_USER_ENC}:${ADMIN_PASS_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
|
|
else
|
|
export DATABASE_URL="mysql://${ADMIN_USER_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
|
|
fi
|
|
|
|
# Keep the CLI from phoning checkpoint.prisma.io for update checks.
|
|
export CHECKPOINT_DISABLE=1
|
|
|
|
if ! "$MYSQL_BIN" "${mysql_args[@]}" -e "SELECT 1" >/dev/null 2>&1; then
|
|
echo "Cannot connect to the database as '$ADMIN_USER'." >&2
|
|
echo "Set PROD_DB_PASS in ~/.env, or pass DB_ADMIN_USER / DB_ADMIN_PASS." >&2
|
|
exit 1
|
|
fi
|
|
|
|
#
|
|
# Collation is explicit because the server default is utf8mb4_unicode_ci while
|
|
# every application table is utf8mb4_general_ci. A table created later without
|
|
# an explicit COLLATE would otherwise inherit the wrong one, changing string
|
|
# comparison and FULLTEXT ranking -- and Prisma neither models collation nor
|
|
# reports the difference in `migrate diff`.
|
|
#
|
|
# For a database that already exists with the wrong default, the one-time
|
|
# non-destructive fix is:
|
|
# ALTER DATABASE application CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
|
|
#
|
|
"$MYSQL_BIN" "${mysql_args[@]}" -e "CREATE DATABASE IF NOT EXISTS \`$DB_NAME\` \
|
|
CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"
|
|
|
|
sql() { "$MYSQL_BIN" "${mysql_args[@]}" -N -B "$DB_NAME" -e "$1" 2>/dev/null; }
|
|
|
|
echo '-------'
|
|
echo "Migrating '$DB_NAME' as '$ADMIN_USER'"
|
|
echo '-------'
|
|
|
|
# --- baseline an existing, pre-Prisma database ---------------------------
|
|
#
|
|
# This is the single most dangerous moment in adopting Prisma. A database that
|
|
# already has the schema but no _prisma_migrations table must have the baseline
|
|
# marked applied, NOT executed -- running 0_init against it would fail on
|
|
# existing tables. Detect and handle it here rather than leaving it as a manual
|
|
# step someone forgets.
|
|
#
|
|
HAS_MIGRATION_TABLE=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='_prisma_migrations'" || echo 0)
|
|
HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment'" || echo 0)
|
|
|
|
if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then
|
|
echo "Existing schema found with no migration history -- baselining."
|
|
|
|
#
|
|
# Report drift BEFORE recording the baseline.
|
|
#
|
|
# `migrate resolve` records a migration without executing it, so baselining
|
|
# a database that does not actually match the schema leaves it permanently
|
|
# short of those objects while Prisma reports everything as applied. That
|
|
# is exactly how production ended up missing scrape_cache: this script
|
|
# previously inferred "the attachment table exists" => "the whole schema
|
|
# exists", which is not the same claim.
|
|
#
|
|
# Later migrations are additive and idempotent, so drift is recoverable --
|
|
# but the operator should see it rather than discover it from a log line
|
|
# days later.
|
|
#
|
|
if ! prisma migrate diff \
|
|
--from-config-datasource \
|
|
--to-schema prisma/schema.prisma \
|
|
--exit-code >/dev/null 2>&1
|
|
then
|
|
echo ""
|
|
echo "NOTE: this database does not yet match prisma/schema.prisma."
|
|
echo " Baselining 0_init anyway, then applying later migrations to"
|
|
echo " converge it. Differences:"
|
|
prisma migrate diff --from-config-datasource --to-schema prisma/schema.prisma \
|
|
| sed 's/^/ /'
|
|
echo ""
|
|
fi
|
|
|
|
echo "Marking 0_init as already applied (NOT executing it)."
|
|
prisma migrate resolve --applied 0_init
|
|
echo ""
|
|
fi
|
|
|
|
# --- apply ---------------------------------------------------------------
|
|
|
|
prisma migrate deploy
|
|
|
|
# --- seed ----------------------------------------------------------------
|
|
|
|
if [ -f prisma/seed.sql ]; then
|
|
echo ""
|
|
echo "Applying reference data (prisma/seed.sql)..."
|
|
"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" < prisma/seed.sql
|
|
fi
|
|
|
|
# --- verify --------------------------------------------------------------
|
|
#
|
|
# Print proof rather than leaving the operator to trust us.
|
|
#
|
|
echo ""
|
|
echo '-------'
|
|
echo 'Verification'
|
|
echo '-------'
|
|
|
|
prisma migrate status
|
|
|
|
echo ""
|
|
"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" --table -e "
|
|
SELECT 'attachment FULLTEXT index' AS check_name,
|
|
IFNULL(MAX(INDEX_TYPE), 'MISSING') AS result
|
|
FROM information_schema.STATISTICS
|
|
WHERE TABLE_SCHEMA = DATABASE() AND INDEX_NAME = 'attachment_text_index'
|
|
UNION ALL
|
|
SELECT 'tables',
|
|
CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> '_prisma_migrations'
|
|
UNION ALL
|
|
SELECT 'attachment_type seed rows',
|
|
CAST(COUNT(*) AS CHAR) FROM attachment_type
|
|
UNION ALL
|
|
-- _prisma_migrations is Prisma's own bookkeeping table and is created with
|
|
-- its default collation; only application tables are checked here.
|
|
SELECT 'non-general_ci app tables (expect 0)',
|
|
CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA = DATABASE()
|
|
AND TABLE_NAME <> '_prisma_migrations'
|
|
AND TABLE_COLLATION <> 'utf8mb4_general_ci';
|
|
"
|
|
|
|
#
|
|
# Assert that everything the application actually needs now exists.
|
|
#
|
|
# This is the check that was missing. `prisma migrate deploy` reporting success
|
|
# only means the recorded migrations ran -- it says nothing about whether the
|
|
# database ended up in the intended shape, which is precisely how production
|
|
# could pass this script and still log
|
|
# "Scraper: DISABLED - missing table scrape_cache ...".
|
|
#
|
|
# Mirrors server/helpers/Schema.js's boot verification, so a green run here
|
|
# means a green boot there.
|
|
#
|
|
FAILURES=0
|
|
require_object() {
|
|
local label="$1" count="$2"
|
|
if [ "${count:-0}" -eq 0 ]; then
|
|
echo "FAILED: missing $label" >&2
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
}
|
|
|
|
require_object "table scrape_cache" "$(sql "SELECT COUNT(*) FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='scrape_cache'")"
|
|
|
|
for column in scrape_state scrape_attempts enqueued_at; do
|
|
require_object "attachment.$column" "$(sql "SELECT COUNT(*) FROM information_schema.COLUMNS
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND COLUMN_NAME='$column'")"
|
|
done
|
|
|
|
require_object "attachment index attachment_scrape_state" "$(sql "SELECT COUNT(*) FROM information_schema.STATISTICS
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND INDEX_NAME='attachment_scrape_state'")"
|
|
|
|
# Checked by TYPE, not just by name: a BTREE index called attachment_text_index
|
|
# would make MATCH..AGAINST return nothing at all, with no error anywhere.
|
|
FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS
|
|
WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'")
|
|
if [ "$FT" != "FULLTEXT" ]; then
|
|
echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2
|
|
echo "Attachment search depends on MATCH..AGAINST and would silently return nothing." >&2
|
|
FAILURES=$((FAILURES + 1))
|
|
fi
|
|
|
|
if [ "$FAILURES" -gt 0 ]; then
|
|
echo "" >&2
|
|
echo "$FAILURES required object(s) missing after migrating -- the scraper will stay disabled." >&2
|
|
echo "This should not happen; check 'prisma migrate status' and the migration SQL." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo '-------'
|
|
echo 'Done. Restart the API so it re-verifies the schema:'
|
|
echo ' pm2 restart NoteServer'
|
|
echo '-------'
|