Testing Migrations for web scraper

deploy.sh

@@ -48,6 +48,60 @@
 #
 # Override the location for a staging box with:  WEBROOT=/var/www/ss-staging ./deploy.sh
 #
+# --- the database account ------------------------------------------------
+#
+# The API connects as a least-privilege account with DML rights only. This is
+# deliberate: the server fetches untrusted URLs for link previews, and a
+# service able to reshape or drop its own tables is a far larger blast radius
+# than that feature warrants.
+#
+#   CREATE USER 'app'@'localhost' IDENTIFIED BY '<password>';
+#   GRANT SELECT, INSERT, UPDATE, DELETE ON application.* TO 'app'@'localhost';
+#   FLUSH PRIVILEGES;
+#
+#   -- deliberately NOT granted: CREATE, ALTER, DROP, INDEX
+#
+# Then set DB_USER=app and DB_PASS=<password> in ~/.env. Note the asymmetry:
+# the admin scripts here (backupDatabase.sh, applyLastBackup.sh,
+# migrateDatabase.sh) still connect as root with $PROD_DB_PASS.
+#
+# DELETE is genuinely required -- the scraper drops shell attachment rows for
+# dead links and purges expired cache entries.
+#
+# --- schema changes ------------------------------------------------------
+#
+# Schema is managed by Prisma, for migrations and scaffolding ONLY -- the
+# running server never imports it and still talks to MariaDB through raw SQL.
+# The schema lives in prisma/schema.prisma; migrations in prisma/migrations/.
+#
+# Because the app holds no DDL, a deploy that introduces schema changes needs
+# one manual step, run once per environment as a database admin:
+#
+#   nix-shell shell.nix --run ./migrateDatabase.sh
+#
+# nix-shell is required: the prisma CLI comes from nixpkgs (prisma_7, 7.8.0),
+# not from node_modules. Alternatively add prisma_7 to the server's
+# environment.systemPackages and call the script directly.
+#
+# It is idempotent, so re-running is safe. On a database that predates Prisma
+# the script detects the missing migration history and baselines it
+# automatically (marking 0_init applied rather than executing it) -- it does
+# NOT need to be done by hand.
+#
+# The API verifies the schema and the grants at boot; if either is missing it
+# logs the remedy and serves normally with link previews disabled.
+#
+# NEVER run `prisma migrate dev`, `prisma migrate reset` or `prisma db push`.
+# The first wants a shadow database; the latter two can drop data.
+#
+# DATABASE_URL is composed by the script from the admin credentials in ~/.env
+# and is never read by the application.
+#
+# Note: _prisma_migrations lives inside the `application` database, so the
+# least-privilege `app` user can write migration history (its grant is
+# database-wide). It holds no DDL, so it cannot reshape tables -- narrowing
+# this further would mean per-table grants, which is a separate project.
+#
 
 set -euo pipefail
 

migrateDatabase.sh

@@ -0,0 +1,292 @@
+#!/usr/bin/env bash
+#
+# Apply database migrations. Run from the repo root, INSIDE nix-shell (that is
+# where the prisma CLI lives).
+#
+#   nix-shell shell.nix --run ./migrateDatabase.sh
+#
+# Works both ways round: it applies pending migrations to an existing database,
+# and builds a complete one from empty. Safe to re-run -- a second run is a
+# no-op. Replaces migrateDatabase.sh.
+#
+# --- why this is a separate step -----------------------------------------
+#
+# The application connects as a least-privilege account (production uses `app`)
+# holding DML rights only -- no CREATE, ALTER or DROP. That is deliberate: the
+# server fetches untrusted URLs for link previews, and a service able to reshape
+# its own tables is a far larger blast radius than that feature warrants.
+#
+# So the app never applies DDL. It verifies the schema at boot
+# (server/helpers/Schema.js) and disables link previews if anything is missing.
+#
+# --- authoring a new migration -------------------------------------------
+#
+# NEVER run `prisma migrate dev`. It can decide the database has drifted and
+# offer to RESET it, which on a box holding a production data restore would
+# destroy real data. It also wants a shadow database. Instead:
+#
+#   1. Edit prisma/schema.prisma
+#   2. NAME=add_whatever
+#      DIR="prisma/migrations/$(date +%Y%m%d%H%M%S)_$NAME"
+#      mkdir -p "$DIR"
+#      prisma migrate diff --from-config-datasource \
+#        --to-schema prisma/schema.prisma --script > "$DIR/migration.sql"
+#   3. READ the generated SQL -- this review is what makes the approach safe
+#   4. ./migrateDatabase.sh
+#
+# Note: Prisma emits `COLLATE utf8mb4_unicode_ci`, but this database is
+# `utf8mb4_general_ci` throughout, and `prisma migrate diff` does NOT report the
+# difference. Fix the collation in any generated migration by hand. This script
+# checks for drift at the end.
+#
+# --- credentials ---------------------------------------------------------
+#
+# Same convention as applyLastBackup.sh / backupDatabase.sh: source ~/.env and
+# connect as root with $PROD_DB_PASS. Falls back to passwordless root (the dev
+# case, where shell.nix also exports DATABASE_URL). Override with:
+#
+#   DB_ADMIN_USER=admin DB_ADMIN_PASS=secret ./migrateDatabase.sh
+#
+set -euo pipefail
+
+REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$REPO"
+
+DB_NAME="application"   # hardcoded in server/config/database.js too
+
+# --- preconditions -------------------------------------------------------
+
+if ! command -v prisma >/dev/null 2>&1; then
+	echo "prisma is not on PATH -- run this inside nix-shell:" >&2
+	echo "  nix-shell shell.nix --run ./migrateDatabase.sh" >&2
+	exit 1
+fi
+
+if ! command -v mariadb >/dev/null 2>&1 && ! command -v mysql >/dev/null 2>&1; then
+	echo "no mariadb/mysql client on PATH -- run this inside nix-shell." >&2
+	exit 1
+fi
+
+if command -v mariadb >/dev/null 2>&1; then MYSQL_BIN="mariadb"; else MYSQL_BIN="mysql"; fi
+
+if ! command -v node >/dev/null 2>&1; then
+	echo "node is not on PATH -- needed to percent-encode the database password." >&2
+	exit 1
+fi
+
+if [ -f "$HOME/.env" ]; then
+	# shellcheck disable=SC1091
+	source "$HOME/.env"
+fi
+
+ADMIN_USER="${DB_ADMIN_USER:-root}"
+ADMIN_PASS="${DB_ADMIN_PASS:-${PROD_DB_PASS:-}}"
+
+mysql_args=(--user "$ADMIN_USER")
+[ -n "$ADMIN_PASS" ] && mysql_args+=(--password="$ADMIN_PASS")
+
+LOCAL_SOCKET="$REPO/.mysql/mysql.sock"
+[ -S "$LOCAL_SOCKET" ] && mysql_args+=(--socket="$LOCAL_SOCKET")
+
+# Prisma needs a URL, and it must carry the ADMIN credentials -- never the
+# app account, which holds no DDL.
+DB_ADMIN_HOST="${DB_ADMIN_HOST:-127.0.0.1}"
+DB_ADMIN_PORT="${DB_ADMIN_PORT:-3306}"
+
+#
+# Percent-encode the password. Production passwords routinely contain @ : / ? #
+# or %, every one of which silently corrupts a URL -- an unencoded `@` would
+# truncate the host and produce a baffling connection error rather than an
+# obvious one.
+#
+encode_component() {
+	node -e 'process.stdout.write(encodeURIComponent(process.argv[1] || ""))' "$1"
+}
+
+ADMIN_USER_ENC="$(encode_component "$ADMIN_USER")"
+
+if [ -n "$ADMIN_PASS" ]; then
+	ADMIN_PASS_ENC="$(encode_component "$ADMIN_PASS")"
+	export DATABASE_URL="mysql://${ADMIN_USER_ENC}:${ADMIN_PASS_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
+else
+	export DATABASE_URL="mysql://${ADMIN_USER_ENC}@${DB_ADMIN_HOST}:${DB_ADMIN_PORT}/${DB_NAME}"
+fi
+
+# Keep the CLI from phoning checkpoint.prisma.io for update checks.
+export CHECKPOINT_DISABLE=1
+
+if ! "$MYSQL_BIN" "${mysql_args[@]}" -e "SELECT 1" >/dev/null 2>&1; then
+	echo "Cannot connect to the database as '$ADMIN_USER'." >&2
+	echo "Set PROD_DB_PASS in ~/.env, or pass DB_ADMIN_USER / DB_ADMIN_PASS." >&2
+	exit 1
+fi
+
+#
+# Collation is explicit because the server default is utf8mb4_unicode_ci while
+# every application table is utf8mb4_general_ci. A table created later without
+# an explicit COLLATE would otherwise inherit the wrong one, changing string
+# comparison and FULLTEXT ranking -- and Prisma neither models collation nor
+# reports the difference in `migrate diff`.
+#
+# For a database that already exists with the wrong default, the one-time
+# non-destructive fix is:
+#   ALTER DATABASE application CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+#
+"$MYSQL_BIN" "${mysql_args[@]}" -e "CREATE DATABASE IF NOT EXISTS \`$DB_NAME\` \
+	CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"
+
+sql() { "$MYSQL_BIN" "${mysql_args[@]}" -N -B "$DB_NAME" -e "$1" 2>/dev/null; }
+
+echo '-------'
+echo "Migrating '$DB_NAME' as '$ADMIN_USER'"
+echo '-------'
+
+# --- baseline an existing, pre-Prisma database ---------------------------
+#
+# This is the single most dangerous moment in adopting Prisma. A database that
+# already has the schema but no _prisma_migrations table must have the baseline
+# marked applied, NOT executed -- running 0_init against it would fail on
+# existing tables. Detect and handle it here rather than leaving it as a manual
+# step someone forgets.
+#
+HAS_MIGRATION_TABLE=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='_prisma_migrations'" || echo 0)
+HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment'" || echo 0)
+
+if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then
+	echo "Existing schema found with no migration history -- baselining."
+
+	#
+	# Report drift BEFORE recording the baseline.
+	#
+	# `migrate resolve` records a migration without executing it, so baselining
+	# a database that does not actually match the schema leaves it permanently
+	# short of those objects while Prisma reports everything as applied. That
+	# is exactly how production ended up missing scrape_cache: this script
+	# previously inferred "the attachment table exists" => "the whole schema
+	# exists", which is not the same claim.
+	#
+	# Later migrations are additive and idempotent, so drift is recoverable --
+	# but the operator should see it rather than discover it from a log line
+	# days later.
+	#
+	if ! prisma migrate diff \
+		--from-config-datasource \
+		--to-schema prisma/schema.prisma \
+		--exit-code >/dev/null 2>&1
+	then
+		echo ""
+		echo "NOTE: this database does not yet match prisma/schema.prisma."
+		echo "      Baselining 0_init anyway, then applying later migrations to"
+		echo "      converge it. Differences:"
+		prisma migrate diff --from-config-datasource --to-schema prisma/schema.prisma \
+			| sed 's/^/        /'
+		echo ""
+	fi
+
+	echo "Marking 0_init as already applied (NOT executing it)."
+	prisma migrate resolve --applied 0_init
+	echo ""
+fi
+
+# --- apply ---------------------------------------------------------------
+
+prisma migrate deploy
+
+# --- seed ----------------------------------------------------------------
+
+if [ -f prisma/seed.sql ]; then
+	echo ""
+	echo "Applying reference data (prisma/seed.sql)..."
+	"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" < prisma/seed.sql
+fi
+
+# --- verify --------------------------------------------------------------
+#
+# Print proof rather than leaving the operator to trust us.
+#
+echo ""
+echo '-------'
+echo 'Verification'
+echo '-------'
+
+prisma migrate status
+
+echo ""
+"$MYSQL_BIN" "${mysql_args[@]}" "$DB_NAME" --table -e "
+	SELECT 'attachment FULLTEXT index' AS check_name,
+		IFNULL(MAX(INDEX_TYPE), 'MISSING') AS result
+	FROM information_schema.STATISTICS
+	WHERE TABLE_SCHEMA = DATABASE() AND INDEX_NAME = 'attachment_text_index'
+	UNION ALL
+	SELECT 'tables',
+		CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> '_prisma_migrations'
+	UNION ALL
+	SELECT 'attachment_type seed rows',
+		CAST(COUNT(*) AS CHAR) FROM attachment_type
+	UNION ALL
+	-- _prisma_migrations is Prisma's own bookkeeping table and is created with
+	-- its default collation; only application tables are checked here.
+	SELECT 'non-general_ci app tables (expect 0)',
+		CAST(COUNT(*) AS CHAR) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA = DATABASE()
+		AND TABLE_NAME <> '_prisma_migrations'
+		AND TABLE_COLLATION <> 'utf8mb4_general_ci';
+"
+
+#
+# Assert that everything the application actually needs now exists.
+#
+# This is the check that was missing. `prisma migrate deploy` reporting success
+# only means the recorded migrations ran -- it says nothing about whether the
+# database ended up in the intended shape, which is precisely how production
+# could pass this script and still log
+# "Scraper: DISABLED - missing table scrape_cache ...".
+#
+# Mirrors server/helpers/Schema.js's boot verification, so a green run here
+# means a green boot there.
+#
+FAILURES=0
+require_object() {
+	local label="$1" count="$2"
+	if [ "${count:-0}" -eq 0 ]; then
+		echo "FAILED: missing $label" >&2
+		FAILURES=$((FAILURES + 1))
+	fi
+}
+
+require_object "table scrape_cache" "$(sql "SELECT COUNT(*) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='scrape_cache'")"
+
+for column in scrape_state scrape_attempts enqueued_at; do
+	require_object "attachment.$column" "$(sql "SELECT COUNT(*) FROM information_schema.COLUMNS
+		WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND COLUMN_NAME='$column'")"
+done
+
+require_object "attachment index attachment_scrape_state" "$(sql "SELECT COUNT(*) FROM information_schema.STATISTICS
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND INDEX_NAME='attachment_scrape_state'")"
+
+# Checked by TYPE, not just by name: a BTREE index called attachment_text_index
+# would make MATCH..AGAINST return nothing at all, with no error anywhere.
+FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS
+	WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'")
+if [ "$FT" != "FULLTEXT" ]; then
+	echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2
+	echo "Attachment search depends on MATCH..AGAINST and would silently return nothing." >&2
+	FAILURES=$((FAILURES + 1))
+fi
+
+if [ "$FAILURES" -gt 0 ]; then
+	echo "" >&2
+	echo "$FAILURES required object(s) missing after migrating -- the scraper will stay disabled." >&2
+	echo "This should not happen; check 'prisma migrate status' and the migration SQL." >&2
+	exit 1
+fi
+
+echo ""
+echo '-------'
+echo 'Done. Restart the API so it re-verifies the schema:'
+echo '  pm2 restart NoteServer'
+echo '-------'

prisma.config.mjs

@@ -0,0 +1,32 @@
+//
+// Prisma CLI configuration.
+//
+// Prisma 7 removed `url` from the datasource block in schema.prisma; the
+// connection string for migrations lives here instead.
+//
+// This file configures the CLI only. The running server does not use Prisma --
+// it talks to MariaDB through server/config/database.js (mysql2) with raw SQL.
+// Prisma is a development and admin tool here, for schema migrations and
+// scaffolding.
+//
+// DATABASE_URL must point at an account with DDL rights. In dev that is the
+// root user exported by shell.nix; in production migrateDatabase.sh composes it
+// from the admin credentials in ~/.env. It must NEVER be the least-privilege
+// `app` account the server runs as, which deliberately holds no CREATE/ALTER.
+//
+// .mjs rather than .js because this project is CommonJS and Prisma's example
+// config is ESM; the explicit extension avoids any ambiguity.
+//
+export default {
+	schema: 'prisma/schema.prisma',
+
+	migrations: {
+		path: 'prisma/migrations',
+	},
+
+	datasource: {
+		// process.env directly rather than env() -- the helper throws when the
+		// variable is unset, which makes the failure harder to read.
+		url: process.env.DATABASE_URL,
+	},
+}

prisma/migrations/0002_scrape_objects_repair/migration.sql

@@ -0,0 +1,74 @@
+--
+-- Repair: converge environments that were baselined before the scrape objects
+-- existed.
+--
+-- WHY THIS EXISTS
+--
+-- The 0_init baseline was generated from a database that already had
+-- scrape_cache and the attachment.scrape_* columns, on the understanding that
+-- every environment was in that state. Production was not. Because
+-- `prisma migrate resolve --applied 0_init` records a migration WITHOUT
+-- executing it, production ended up with 0_init marked applied while those
+-- objects were never created -- so the scraper logged:
+--
+--   Scraper: DISABLED - missing table scrape_cache, attachment.scrape_state, ...
+--
+-- Re-running 0_init is not an option (it would try to CREATE TABLE over ~2200
+-- rows of live data), so this additive, idempotent migration brings any
+-- environment up to the intended schema instead.
+--
+-- Every statement uses IF NOT EXISTS, so this is a clean no-op on environments
+-- that already had these objects. MariaDB 11.4 supports that syntax for columns
+-- and indexes; both dev and production are MariaDB, so this is safe here. It
+-- would NOT be valid on MySQL 8.
+--
+-- The definitions below are identical to those in 0_init -- deliberately so,
+-- because the end state must be the same whichever path an environment took.
+--
+
+-- One row per scraped URL, shared across users and notes.
+--
+-- url_hash carries the UNIQUE index rather than `url` because a UNIQUE on
+-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit.
+--
+-- Collation is explicit: Prisma renders utf8mb4_unicode_ci, but every table in
+-- this database is utf8mb4_general_ci, and `prisma migrate diff` does not
+-- report the difference.
+CREATE TABLE IF NOT EXISTS `scrape_cache` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `url_hash` CHAR(64) NOT NULL,
+    `url` VARCHAR(1000) NOT NULL DEFAULT '',
+    `canonical_url` VARCHAR(1000) NULL,
+    `status` SMALLINT NOT NULL DEFAULT 0,
+    `title` VARCHAR(512) NULL,
+    `site_name` VARCHAR(255) NULL,
+    `description` MEDIUMTEXT NULL,
+    `keywords` VARCHAR(512) NULL,
+    `search_text` MEDIUMTEXT NULL,
+    `image_url` VARCHAR(1000) NULL,
+    `file_location` VARCHAR(500) NULL,
+    `etag` VARCHAR(255) NULL,
+    `last_modified` VARCHAR(128) NULL,
+    `content_type` VARCHAR(128) NULL,
+    `fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
+    `failure_count` INTEGER NOT NULL DEFAULT 0,
+    `last_error` VARCHAR(255) NULL,
+    `fetched_at` INTEGER NULL,
+    `expires_at` INTEGER NULL,
+
+    UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
+    INDEX `scrape_cache_expires_at`(`expires_at`),
+    INDEX `scrape_cache_fetch_state`(`fetch_state`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- Per-attachment scrape lifecycle, so the in-memory queue survives a restart.
+--
+-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
+-- DEFAULT 2 is load-bearing -- it keeps every pre-existing attachment row
+-- (~2200 in production) out of the scraper's boot recovery sweep.
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state`    TINYINT NOT NULL DEFAULT 2;
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` TINYINT NOT NULL DEFAULT 0;
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at`     INTEGER NULL;
+
+CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);

prisma/migrations/0_init/migration.sql

@@ -0,0 +1,211 @@
+--
+-- Baseline migration: the schema as it existed when Prisma was adopted.
+--
+-- Generated with:
+--   prisma migrate diff --from-empty --to-schema prisma/schema.prisma --script
+--
+-- HAND-CORRECTED, and it matters: Prisma renders `COLLATE utf8mb4_unicode_ci`
+-- for MySQL, but every live table is `utf8mb4_general_ci`. Prisma does not
+-- model collation and its own `migrate diff` does NOT report the difference, so
+-- without this correction a database rebuilt from empty would silently differ
+-- from production in string comparison, sorting and FULLTEXT ranking.
+-- If you regenerate this file, re-apply the collation fix. There is a test for
+-- it in server/tests/schema/migrations.test.js.
+--
+-- Things Prisma cannot express, which therefore need handling elsewhere:
+--   * the DATABASE-level default collation -- set by migrateDatabase.sh when it
+--     creates the database, since tables created later without an explicit
+--     COLLATE would otherwise inherit the server default
+--   * ENGINE=InnoDB (relies on the server default, which is InnoDB)
+--   * AUTO_INCREMENT start values (cosmetic; a fresh database starting at 1 is
+--     correct)
+--   * the COMMENT on note_raw_text.text
+--   * row data -- see prisma/seed.sql for the attachment_type lookup rows
+--
+-- Context carried over from the migration this replaces: scrape_cache.url_hash
+-- carries the UNIQUE index rather than `url` because a UNIQUE on
+-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit;
+-- and attachment.scrape_state DEFAULT 2 ("done") is load-bearing -- it keeps
+-- pre-existing attachment rows out of the scraper's boot recovery sweep.
+--
+-- On environments that already had this schema (dev and production), this file
+-- is NOT executed -- it is marked applied with `prisma migrate resolve`.
+--
+
+-- CreateTable
+CREATE TABLE `attachment` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `note_id` INTEGER UNSIGNED NULL,
+    `user_id` INTEGER UNSIGNED NOT NULL,
+    `attachment_type` INTEGER UNSIGNED NULL,
+    `text` MEDIUMTEXT NULL,
+    `url` VARCHAR(1000) NULL DEFAULT '',
+    `last_indexed` INTEGER NULL,
+    `file_location` VARCHAR(500) NULL,
+    `visible` TINYINT NULL DEFAULT 1,
+    `scrape_state` TINYINT NOT NULL DEFAULT 2,
+    `scrape_attempts` TINYINT NOT NULL DEFAULT 0,
+    `enqueued_at` INTEGER NULL,
+
+    INDEX `attachment_scrape_state`(`scrape_state`, `enqueued_at`),
+    INDEX `attachment_type`(`attachment_type`),
+    INDEX `note_id`(`note_id`),
+    INDEX `user_id`(`user_id`),
+    FULLTEXT INDEX `attachment_text_index`(`text`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `attachment_type` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `mime_type` VARCHAR(500) NULL,
+    `extension` VARCHAR(10) NULL,
+    `name` VARCHAR(50) NULL,
+
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `note` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `user_id` INTEGER UNSIGNED NULL,
+    `note_raw_text_id` INTEGER NULL,
+    `opened` INTEGER NULL,
+    `created` INTEGER NULL,
+    `pinned` INTEGER NULL DEFAULT 0,
+    `archived` INTEGER NULL DEFAULT 0,
+    `trashed` TINYINT NULL DEFAULT 0,
+    `color` VARCHAR(500) NULL,
+    `shared` INTEGER NULL DEFAULT 0,
+    `share_user_id` INTEGER NULL,
+    `quick_note` INTEGER NULL DEFAULT 0,
+    `indexed` TINYINT NULL DEFAULT 0,
+    `snippet` VARCHAR(1300) NULL,
+    `snippet_salt` VARCHAR(1300) NULL,
+    `encrypted_share_password_key` VARCHAR(1300) NULL,
+
+    INDEX `user_note`(`user_id`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `note_raw_text` (
+    `id` INTEGER NOT NULL AUTO_INCREMENT,
+    `text` LONGTEXT NULL,
+    `updated` BIGINT NULL,
+    `salt` VARCHAR(1300) NULL,
+    `last_decrypted_date` INTEGER NULL,
+    `decrypt_attempts_count` INTEGER NULL DEFAULT 0,
+
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `note_tag` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `note_id` INTEGER UNSIGNED NULL,
+    `tag_id` INTEGER UNSIGNED NULL,
+    `user_id` INTEGER UNSIGNED NULL,
+
+    INDEX `note_id`(`note_id`),
+    INDEX `tag_id`(`tag_id`),
+    INDEX `user_id`(`user_id`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `scrape_cache` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `url_hash` CHAR(64) NOT NULL,
+    `url` VARCHAR(1000) NOT NULL DEFAULT '',
+    `canonical_url` VARCHAR(1000) NULL,
+    `status` SMALLINT NOT NULL DEFAULT 0,
+    `title` VARCHAR(512) NULL,
+    `site_name` VARCHAR(255) NULL,
+    `description` MEDIUMTEXT NULL,
+    `keywords` VARCHAR(512) NULL,
+    `search_text` MEDIUMTEXT NULL,
+    `image_url` VARCHAR(1000) NULL,
+    `file_location` VARCHAR(500) NULL,
+    `etag` VARCHAR(255) NULL,
+    `last_modified` VARCHAR(128) NULL,
+    `content_type` VARCHAR(128) NULL,
+    `fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
+    `failure_count` INTEGER NOT NULL DEFAULT 0,
+    `last_error` VARCHAR(255) NULL,
+    `fetched_at` INTEGER NULL,
+    `expires_at` INTEGER NULL,
+
+    UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
+    INDEX `scrape_cache_expires_at`(`expires_at`),
+    INDEX `scrape_cache_fetch_state`(`fetch_state`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `tag` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `text` VARCHAR(400) NULL,
+    `hash` INTEGER NULL,
+
+    INDEX `hash`(`hash`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `user` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `username` VARCHAR(150) NOT NULL DEFAULT '',
+    `password` TEXT NOT NULL,
+    `salt` VARCHAR(500) NOT NULL DEFAULT '',
+    `iterations` INTEGER NOT NULL,
+    `last_login` DATETIME(0) NULL,
+    `created` DATETIME(0) NULL,
+    `two_fa_secret` VARCHAR(50) NULL,
+    `two_fa_enabled` TINYINT NULL DEFAULT 0,
+    `two_fa_every_login` TINYINT NULL DEFAULT 0,
+    `pushkey` VARCHAR(100) NULL,
+
+    UNIQUE INDEX `id_UNIQUE`(`id`),
+    UNIQUE INDEX `username_UNIQUE`(`username`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `user_active_session` (
+    `id` INTEGER NOT NULL AUTO_INCREMENT,
+    `salt` VARCHAR(100) NULL,
+    `encrypted_master_password` VARCHAR(500) NULL,
+    `created` INTEGER NULL,
+    `uses` INTEGER NULL,
+    `user_hash` VARCHAR(500) NULL,
+    `active` TINYINT NULL DEFAULT 1,
+    `session_id` VARCHAR(100) NULL,
+    `last_used` INTEGER NULL,
+
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `user_encrypted_search_index` (
+    `id` INTEGER NOT NULL AUTO_INCREMENT,
+    `user_id` INTEGER NULL,
+    `salt` VARCHAR(100) NULL,
+    `index` LONGTEXT NULL,
+    `last_update` INTEGER NULL,
+
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- CreateTable
+CREATE TABLE `user_key` (
+    `id` INTEGER NOT NULL AUTO_INCREMENT,
+    `user_id` INTEGER NULL,
+    `salt` VARCHAR(1300) NULL,
+    `key` VARCHAR(5000) NULL,
+    `created` INTEGER NULL,
+    `public_key` VARCHAR(5000) NULL,
+    `private_key_encrypted` VARCHAR(5000) NULL,
+
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (e.g., Git)
+provider = "mysql"

prisma/schema.prisma

@@ -0,0 +1,151 @@
+datasource db {
+  provider = "mysql"
+}
+
+model attachment {
+  id              Int     @id @default(autoincrement()) @db.UnsignedInt
+  note_id         Int?    @db.UnsignedInt
+  user_id         Int     @db.UnsignedInt
+  attachment_type Int?    @db.UnsignedInt
+  text            String? @db.MediumText
+  url             String? @default("") @db.VarChar(1000)
+  last_indexed    Int?
+  file_location   String? @db.VarChar(500)
+  visible         Int?    @default(1) @db.TinyInt
+  scrape_state    Int     @default(2) @db.TinyInt
+  scrape_attempts Int     @default(0) @db.TinyInt
+  enqueued_at     Int?
+
+  @@index([scrape_state, enqueued_at], map: "attachment_scrape_state")
+  @@index([attachment_type], map: "attachment_type")
+  @@index([note_id], map: "note_id")
+  @@index([user_id], map: "user_id")
+  @@fulltext([text], map: "attachment_text_index")
+}
+
+model attachment_type {
+  id        Int     @id @default(autoincrement()) @db.UnsignedInt
+  mime_type String? @db.VarChar(500)
+  extension String? @db.VarChar(10)
+  name      String? @db.VarChar(50)
+}
+
+model note {
+  id                           Int     @id @default(autoincrement()) @db.UnsignedInt
+  user_id                      Int?    @db.UnsignedInt
+  note_raw_text_id             Int?
+  opened                       Int?
+  created                      Int?
+  pinned                       Int?    @default(0)
+  archived                     Int?    @default(0)
+  trashed                      Int?    @default(0) @db.TinyInt
+  color                        String? @db.VarChar(500)
+  shared                       Int?    @default(0)
+  share_user_id                Int?
+  quick_note                   Int?    @default(0)
+  indexed                      Int?    @default(0) @db.TinyInt
+  snippet                      String? @db.VarChar(1300)
+  snippet_salt                 String? @db.VarChar(1300)
+  encrypted_share_password_key String? @db.VarChar(1300)
+
+  @@index([user_id], map: "user_note")
+}
+
+/// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments
+model note_raw_text {
+  id                     Int     @id @default(autoincrement())
+  text                   String? @db.LongText
+  updated                BigInt?
+  salt                   String? @db.VarChar(1300)
+  last_decrypted_date    Int?
+  decrypt_attempts_count Int?    @default(0)
+}
+
+model note_tag {
+  id      Int  @id @default(autoincrement()) @db.UnsignedInt
+  note_id Int? @db.UnsignedInt
+  tag_id  Int? @db.UnsignedInt
+  user_id Int? @db.UnsignedInt
+
+  @@index([note_id], map: "note_id")
+  @@index([tag_id], map: "tag_id")
+  @@index([user_id], map: "user_id")
+}
+
+model scrape_cache {
+  id            Int     @id @default(autoincrement()) @db.UnsignedInt
+  url_hash      String  @unique(map: "scrape_cache_url_hash") @db.Char(64)
+  url           String  @default("") @db.VarChar(1000)
+  canonical_url String? @db.VarChar(1000)
+  status        Int     @default(0) @db.SmallInt
+  title         String? @db.VarChar(512)
+  site_name     String? @db.VarChar(255)
+  description   String? @db.MediumText
+  keywords      String? @db.VarChar(512)
+  search_text   String? @db.MediumText
+  image_url     String? @db.VarChar(1000)
+  file_location String? @db.VarChar(500)
+  etag          String? @db.VarChar(255)
+  last_modified String? @db.VarChar(128)
+  content_type  String? @db.VarChar(128)
+  fetch_state   String  @default("pending") @db.VarChar(20)
+  failure_count Int     @default(0)
+  last_error    String? @db.VarChar(255)
+  fetched_at    Int?
+  expires_at    Int?
+
+  @@index([expires_at], map: "scrape_cache_expires_at")
+  @@index([fetch_state], map: "scrape_cache_fetch_state")
+}
+
+model tag {
+  id   Int     @id @default(autoincrement()) @db.UnsignedInt
+  text String? @db.VarChar(400)
+  hash Int?
+
+  @@index([hash], map: "hash")
+}
+
+model user {
+  id                 Int       @id @unique(map: "id_UNIQUE") @default(autoincrement()) @db.UnsignedInt
+  username           String    @unique(map: "username_UNIQUE") @default("") @db.VarChar(150)
+  password           String    @db.Text
+  salt               String    @default("") @db.VarChar(500)
+  iterations         Int
+  last_login         DateTime? @db.DateTime(0)
+  created            DateTime? @db.DateTime(0)
+  two_fa_secret      String?   @db.VarChar(50)
+  two_fa_enabled     Int?      @default(0) @db.TinyInt
+  two_fa_every_login Int?      @default(0) @db.TinyInt
+  pushkey            String?   @db.VarChar(100)
+}
+
+model user_active_session {
+  id                        Int     @id @default(autoincrement())
+  salt                      String? @db.VarChar(100)
+  encrypted_master_password String? @db.VarChar(500)
+  created                   Int?
+  uses                      Int?
+  user_hash                 String? @db.VarChar(500)
+  active                    Int?    @default(1) @db.TinyInt
+  session_id                String? @db.VarChar(100)
+  last_used                 Int?
+}
+
+model user_encrypted_search_index {
+  id          Int     @id @default(autoincrement())
+  user_id     Int?
+  salt        String? @db.VarChar(100)
+  index       String? @db.LongText
+  last_update Int?
+}
+
+model user_key {
+  id                    Int     @id @default(autoincrement())
+  user_id               Int?
+  salt                  String? @db.VarChar(1300)
+  key                   String? @db.VarChar(5000)
+  created               Int?
+  public_key            String? @db.VarChar(5000)
+  private_key_encrypted String? @db.VarChar(5000)
+}

prisma/seed.sql

@@ -0,0 +1,24 @@
+--
+-- Reference data required for a working database.
+--
+-- Applied by ./migrateDatabase.sh after migrations. Kept out of the migration
+-- history on purpose: environments that were baselined with
+-- `prisma migrate resolve` never execute 0_init, so seed rows placed inside it
+-- would silently never exist there.
+--
+-- Idempotent, so re-running is harmless.
+--
+
+-- attachment_type is a lookup table the application joins against.
+-- attachment_type = 1 is hardcoded for scraped URLs, 2 for uploaded images,
+-- 3 for PDFs (see server/models/Attachment.js). Without these rows a freshly
+-- scaffolded database creates attachments that reference nothing.
+INSERT INTO `attachment_type` (`id`, `mime_type`, `extension`, `name`) VALUES
+  (1, NULL, NULL,   'URL'),
+  (2, NULL, '.jpg', 'Image - Jpg'),
+  (3, NULL, '.pdf', 'PDF'),
+  (4, NULL, '.doc', 'DOC')
+ON DUPLICATE KEY UPDATE
+  `mime_type` = VALUES(`mime_type`),
+  `extension` = VALUES(`extension`),
+  `name`      = VALUES(`name`);

server/helpers/Schema.js

@@ -1,58 +1,68 @@
 //
-// Idempotent schema convergence for the scraper tables.
+// Verification of the scraper's schema and privileges. Issues NO DDL.
 //
-// There is no migration framework in this project and no schema file in git,
+// This module used to converge the schema at boot with CREATE TABLE / ALTER
-// so this runs at boot, checks information_schema for each object, and applies
+// TABLE. That worked only because dev connects as root (shell.nix exports
-// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
+// DB_USER=root); on production, which runs as a least-privilege `app` user, it
+// failed with:
 //
-// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
+//   CREATE command denied to user 'app'@'localhost' for table scrape_cache
-// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
+//
-// every ALTER is guarded by an explicit existence check instead.
+// The restricted account is correct - the server ingests untrusted URLs and
+// should not be able to reshape its own tables. So DDL moved out of the
+// application entirely: an admin runs ./migrateDatabase.sh once, and this
+// module only checks that the result is present and usable.
+//
+// Note on why the DDL is REMOVED rather than guarded by an existence check:
+// MariaDB evaluates the CREATE privilege before IF NOT EXISTS, so
+// `CREATE TABLE IF NOT EXISTS` would keep erroring for a DML-only user on every
+// boot even once the table exists.
 //
 
 const db = require('@config/database')
 
 let Schema = module.exports = {}
 
-const SCRAPE_CACHE_TABLE = `
+const REQUIRED_TABLE = 'scrape_cache'
-	CREATE TABLE IF NOT EXISTS scrape_cache (
+const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at']
-		id            int(10) unsigned NOT NULL AUTO_INCREMENT,
+const REQUIRED_INDEX = 'attachment_scrape_state'
-		url_hash      char(64) NOT NULL,
-		url           varchar(1000) NOT NULL DEFAULT '',
-		canonical_url varchar(1000) DEFAULT NULL,
-		status        smallint(6) NOT NULL DEFAULT 0,
-		title         varchar(512) DEFAULT NULL,
-		site_name     varchar(255) DEFAULT NULL,
-		description   mediumtext DEFAULT NULL,
-		keywords      varchar(512) DEFAULT NULL,
-		search_text   mediumtext DEFAULT NULL,
-		image_url     varchar(1000) DEFAULT NULL,
-		file_location varchar(500) DEFAULT NULL,
-		etag          varchar(255) DEFAULT NULL,
-		last_modified varchar(128) DEFAULT NULL,
-		content_type  varchar(128) DEFAULT NULL,
-		fetch_state   varchar(20) NOT NULL DEFAULT 'pending',
-		failure_count int(11) NOT NULL DEFAULT 0,
-		last_error    varchar(255) DEFAULT NULL,
-		fetched_at    int(11) DEFAULT NULL,
-		expires_at    int(11) DEFAULT NULL,
-		PRIMARY KEY (id),
-		UNIQUE KEY scrape_cache_url_hash (url_hash),
-		KEY scrape_cache_expires_at (expires_at),
-		KEY scrape_cache_fetch_state (fetch_state)
-	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
-`
 
-// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
+//
-const ATTACHMENT_COLUMNS = [
+// The FULLTEXT index backing Attachment.textSearch's MATCH..AGAINST.
-	{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
+//
-	{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
+// Checked by TYPE, not just existence: Prisma's `@@fulltext` is a preview
-	{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
+// feature and MariaDB is not among its documented databases, so a future
-]
+// migration could plausibly emit a plain BTREE index of the same name. That
+// would make attachment search return nothing at all, with no error anywhere -
+// so it is worth asserting the type explicitly on every boot.
+//
+const REQUIRED_FULLTEXT = { table: 'attachment', index: 'attachment_text_index' }
 
-const ATTACHMENT_INDEXES = [
+// The scraper writes and deletes, not just reads. DELETE is genuinely needed:
-	{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
+// terminal failures drop the shell attachment row (ScrapeQueue), and
-]
+// ScrapeCache.purgeExpired removes expired rows.
+const REQUIRED_PRIVILEGES = ['SELECT', 'INSERT', 'UPDATE', 'DELETE']
+
+// Set by verifyScrapeSchema so the per-request paths can check a flag rather
+// than querying information_schema on every note save.
+let ready = false
+let lastResult = null
+
+Schema.isScrapeSchemaReady = () => ready
+Schema.lastVerification = () => lastResult
+
+// Test seam.
+Schema.__setReady = (value) => { ready = !!value }
+Schema.__reset = () => { ready = false; lastResult = null }
+
+const tableExists = async (table) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.TABLES
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? LIMIT 1`,
+		[table]
+	)
+	return rows.length > 0
+}
 
 const columnExists = async (table, column) => {
 	const [rows] = await db.promise().query(
@@ -63,6 +73,19 @@ const columnExists = async (table, column) => {
 	return rows.length > 0
 }
 
+//
+// Does this index exist AND is it actually FULLTEXT?
+//
+const fulltextIndexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
+			AND INDEX_TYPE = 'FULLTEXT' LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+
 const indexExists = async (table, index) => {
 	const [rows] = await db.promise().query(
 		`SELECT 1 FROM information_schema.STATISTICS
@@ -71,34 +94,146 @@ const indexExists = async (table, index) => {
 	)
 	return rows.length > 0
 }
+
+Schema.__tableExists = tableExists
 Schema.__columnExists = columnExists
 Schema.__indexExists = indexExists
+Schema.__fulltextIndexExists = fulltextIndexExists
 
 //
-// Safe to call repeatedly. Returns the list of statements it actually applied,
+// Parse SHOW GRANTS output into the set of privileges that apply to this
-// so boot logging says something useful on the one run that matters.
+// database. Any user may run SHOW GRANTS FOR CURRENT_USER(), so this needs no
+// privileges of its own.
 //
-Schema.ensureScrapeSchema = async () => {
+// Grant lines look like:
-	const applied = []
+//   GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION
+//   GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`
+//   GRANT SELECT ON `application`.`attachment` TO `app`@`localhost`
+//   GRANT USAGE ON *.* TO `app`@`localhost`
+//
+const PRIVILEGE_PATTERN = /^GRANT\s+(.+?)\s+ON\s+(\S+)\s+TO\s/i
 
-	await db.promise().query(SCRAPE_CACHE_TABLE)
+Schema.parseGrants = (lines, database, tables) => {
+	const granted = new Set()
 
-	for (const column of ATTACHMENT_COLUMNS) {
+	// Scope strings that cover our tables. Backticks are optional in output.
-		if (await columnExists('attachment', column.name)) { continue }
+	const strip = (value) => value.replace(/`/g, '')
-		await db.promise().query(column.ddl)
+	const relevantScopes = new Set([
-		applied.push(`attachment.${column.name}`)
+		'*.*',
+		`${database}.*`,
+		...tables.map(table => `${database}.${table}`),
+	])
+
+	for (const line of lines) {
+		const match = String(line).match(PRIVILEGE_PATTERN)
+		if (!match) { continue }
+
+		const scope = strip(match[2])
+		if (!relevantScopes.has(scope)) { continue }
+
+		// Drop any column-level qualifiers, e.g. "SELECT (id, url)".
+		const privileges = match[1].replace(/\s*\([^)]*\)/g, '')
+
+		for (const privilege of privileges.split(',')) {
+			const name = privilege.trim().toUpperCase()
+			if (!name || name === 'USAGE') { continue }
+
+			if (name === 'ALL PRIVILEGES' || name === 'ALL') {
+				REQUIRED_PRIVILEGES.forEach(required => granted.add(required))
+				continue
 			}
 
-	// Index last: it references the columns above.
+			granted.add(name)
-	for (const index of ATTACHMENT_INDEXES) {
+		}
-		if (await indexExists('attachment', index.name)) { continue }
-		await db.promise().query(index.ddl)
-		applied.push(`attachment KEY ${index.name}`)
 	}
 
-	if (applied.length) {
+	return granted
-		console.log('[schema] applied scrape schema changes:', applied.join(', '))
+}
+
+const checkPrivileges = async () => {
+	let lines = []
+
+	try {
+		const [rows] = await db.promise().query('SHOW GRANTS FOR CURRENT_USER()')
+		// Each row is an object with a single, server-named column.
+		lines = rows.map(row => Object.values(row)[0])
+	} catch (error) {
+		// If we cannot read our own grants, do not block on a guess - the
+		// object checks above are the real gate.
+		console.log('[schema] could not read grants, skipping privilege check:', error.message)
+		return { missing: [], database: null, currentUser: null }
+	}
+
+	const [dbRows] = await db.promise().query('SELECT DATABASE() AS name, CURRENT_USER() AS who')
+	const database = dbRows[0] && dbRows[0].name
+	const currentUser = dbRows[0] && dbRows[0].who
+
+	const granted = Schema.parseGrants(lines, database, ['attachment', REQUIRED_TABLE])
+
+	return {
+		missing: REQUIRED_PRIVILEGES.filter(privilege => !granted.has(privilege)),
+		database,
+		currentUser,
+	}
+}
+
+// 'app@localhost' -> "'app'@'localhost'", so the remedy is copy-pasteable.
+const quoteUser = (currentUser) => {
+	if (!currentUser) { return '<app user>' }
+
+	const [name, host] = String(currentUser).split('@')
+	return `'${name}'@'${host || 'localhost'}'`
+}
+
+//
+// Check that everything the scraper needs is present. Never mutates anything.
+//
+// Returns { ok, missingObjects, missingPrivileges, remedy }.
+//
+Schema.verifyScrapeSchema = async () => {
+	const missingObjects = []
+
+	if (!await tableExists(REQUIRED_TABLE)) {
+		missingObjects.push(`table ${REQUIRED_TABLE}`)
+	}
+
+	for (const column of REQUIRED_COLUMNS) {
+		if (!await columnExists('attachment', column)) {
+			missingObjects.push(`attachment.${column}`)
+		}
+	}
+
+	if (!await indexExists('attachment', REQUIRED_INDEX)) {
+		missingObjects.push(`attachment index ${REQUIRED_INDEX}`)
+	}
+
+	if (!await fulltextIndexExists(REQUIRED_FULLTEXT.table, REQUIRED_FULLTEXT.index)) {
+		missingObjects.push(`FULLTEXT index ${REQUIRED_FULLTEXT.table}.${REQUIRED_FULLTEXT.index}`)
+	}
+
+	const privileges = await checkPrivileges()
+	const missingPrivileges = privileges.missing || []
+
+	const remedy = []
+	if (missingObjects.length) {
+		remedy.push('run ./migrateDatabase.sh as a database admin')
+	}
+	if (missingPrivileges.length) {
+		// Name the real database and user so this can be pasted straight into
+		// a mysql prompt rather than decoded first.
+		const target = `${privileges.database || '<database>'}.*`
+		remedy.push(
+			`GRANT ${missingPrivileges.join(', ')} ON ${target} TO ${quoteUser(privileges.currentUser)}`
+		)
+	}
+
+	lastResult = {
+		ok: missingObjects.length === 0 && missingPrivileges.length === 0,
+		missingObjects,
+		missingPrivileges,
+		remedy: remedy.join('; '),
 	}
+	ready = lastResult.ok
 
-	return applied
+	return lastResult
 }

server/index.js

@@ -348,16 +348,34 @@ app.use('/api/metric-tracking', metricTracking)
 //
 // Scraper startup.
 //
-// ensureScrapeSchema() converges the scrape_cache table and the attachment
+// The application deliberately holds no DDL privileges, so it does not create
-// lifecycle columns. It is idempotent and runs before the listener binds, so
+// its own schema - verifyScrapeSchema() only checks that an admin has already
-// there is no window where requests arrive against a schema that is mid-change.
+// run ./migrateDatabase.sh and that the required DML grants are present.
-// There is no migration framework in this project - see server/helpers/Schema.js.
+//
+// When anything is missing the scraper stays off rather than failing an INSERT
+// on every note save. The listener binds either way: the API must not depend on
+// link previews being available.
 //
 const Schema = require('@helpers/Schema')
 const ScrapeQueue = require('@workers/ScrapeQueue')
 
-Schema.ensureScrapeSchema()
+Schema.verifyScrapeSchema()
-	.then(() => {
+	.then(result => {
+		if (!result.ok) {
+			const missing = [
+				...result.missingObjects,
+				...result.missingPrivileges.map(privilege => `privilege ${privilege}`),
+			].join(', ')
+
+			console.log(`Scraper: DISABLED - missing ${missing}. To fix: ${result.remedy}`)
+			return null
+		}
+
+		// Say so positively. Previously a healthy scraper logged nothing, so
+		// "started" and "silently disabled" looked identical in pm2 logs --
+		// which is how a dead scraper went unnoticed on production.
+		console.log('Scraper: ready')
+
 		ScrapeQueue.start()
 
 		// The queue lives in memory, so a pm2 reload loses whatever was in it.
@@ -372,7 +390,7 @@ Schema.ensureScrapeSchema()
 	})
 	.catch(error => {
 		// A scraper that cannot start must not stop the API from serving.
-		console.log('Scraper: startup failed, link previews disabled this boot:', error.message)
+		console.log('Scraper: startup check failed, link previews disabled this boot:', error.message)
 	})
 	.finally(() => {
 		//Output running status

server/migrations/0001_scrape_cache_and_state.sql

@@ -1,51 +0,0 @@
---
--- Scrape cache + per-attachment scrape lifecycle.
---
--- This file is the human-readable record. It is NOT applied by hand: the same
--- statements live in server/helpers/Schema.js and are applied idempotently at
--- boot after an information_schema existence check, so dev and prod converge
--- without a migration runner.
---
--- Portability: dev is MariaDB 11.4, prod is MySQL 8. Nothing here may use
--- ADD COLUMN IF NOT EXISTS (MariaDB-only).
---
-
--- One row per URL, shared across users and notes. url_hash carries the UNIQUE
--- index because a UNIQUE on varchar(1000) utf8mb4 is 4000 bytes and exceeds
--- InnoDB's 3072-byte key limit.
-CREATE TABLE IF NOT EXISTS `scrape_cache` (
-  `id`            int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `url_hash`      char(64) NOT NULL,
-  `url`           varchar(1000) NOT NULL DEFAULT '',
-  `canonical_url` varchar(1000) DEFAULT NULL,
-  `status`        smallint(6) NOT NULL DEFAULT 0,
-  `title`         varchar(512) DEFAULT NULL,
-  `site_name`     varchar(255) DEFAULT NULL,
-  `description`   mediumtext DEFAULT NULL,
-  `keywords`      varchar(512) DEFAULT NULL,
-  `search_text`   mediumtext DEFAULT NULL,
-  `image_url`     varchar(1000) DEFAULT NULL,
-  `file_location` varchar(500) DEFAULT NULL,
-  `etag`          varchar(255) DEFAULT NULL,
-  `last_modified` varchar(128) DEFAULT NULL,
-  `content_type`  varchar(128) DEFAULT NULL,
-  `fetch_state`   varchar(20) NOT NULL DEFAULT 'pending',
-  `failure_count` int(11) NOT NULL DEFAULT 0,
-  `last_error`    varchar(255) DEFAULT NULL,
-  `fetched_at`    int(11) DEFAULT NULL,
-  `expires_at`    int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
-  KEY `scrape_cache_expires_at` (`expires_at`),
-  KEY `scrape_cache_fetch_state` (`fetch_state`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
--- Per-attachment lifecycle, so an in-memory queue survives a restart.
---
--- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
--- DEFAULT 2 is load-bearing: it keeps the ~2200 pre-existing rows out of the
--- recovery sweep on first boot.
-ALTER TABLE `attachment` ADD COLUMN `scrape_state`    tinyint(4) NOT NULL DEFAULT 2;
-ALTER TABLE `attachment` ADD COLUMN `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
-ALTER TABLE `attachment` ADD COLUMN `enqueued_at`     int(11) DEFAULT NULL;
-ALTER TABLE `attachment` ADD KEY `attachment_scrape_state` (`scrape_state`, `enqueued_at`);

server/models/Attachment.js

@@ -288,11 +288,18 @@ Attachment.generateThumbnail = (fileName) => {
 Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
 
 	const ScrapeQueue = require('@workers/ScrapeQueue')
+	const Schema = require('@helpers/Schema')
 
 	// Fixed: the old version called resolve() without returning, so an empty
 	// note fell through and ran the DB lookup anyway.
 	if (!noteText || noteText.length == 0) { return Promise.resolve('') }
 
+	// When the scrape schema is absent the new columns do not exist, so
+	// creating a pending row would fail on every save. The diff and soft-delete
+	// below still run - they touch no new columns - so removing a link from a
+	// note behaves exactly as before.
+	const scrapeAvailable = Schema.isScrapeSchemaReady()
+
 	let knownText = ''
 
 	return Attachment.urlForNote(userId, noteId)
@@ -320,6 +327,10 @@ Attachment.scanTextForWebsites = (userId, noteId, noteText) => {
 
 			if (!remaining.length) { return knownText }
 
+			// No schema, no previews. Return what is already cached rather than
+			// attempting inserts that cannot succeed.
+			if (!scrapeAvailable) { return knownText }
+
 			// Create a shell row per new URL, then hand off to the worker.
 			return Promise.all(remaining.map(url =>
 				Attachment.createPendingUrlAttachment(userId, noteId, url)
@@ -378,8 +389,10 @@ Attachment.createPendingUrlAttachment = (userId, noteId, url) => {
 Attachment.scrapeUrlsCreateAttachments = (userId, noteId, foundUrls) => {
 
 	const ScrapeQueue = require('@workers/ScrapeQueue')
+	const Schema = require('@helpers/Schema')
 
 	if (!foundUrls || !foundUrls.length) { return Promise.resolve('') }
+	if (!Schema.isScrapeSchemaReady()) { return Promise.resolve('') }
 
 	return Promise.all(foundUrls.map(url =>
 		Attachment.createPendingUrlAttachment(userId, noteId, url)
@@ -562,6 +575,11 @@ Attachment.getPushkeyBookmarklet = (userId) => {
 Attachment.pushUrl = (pushkey, url) => {
 
 	const ScrapeQueue = require('@workers/ScrapeQueue')
+	const Schema = require('@helpers/Schema')
+
+	// Preserve the bookmarklet contract - resolve truthy, leaking nothing -
+	// rather than failing an insert against columns that do not exist.
+	if (!Schema.isScrapeSchemaReady()) { return Promise.resolve(true) }
 
 	let userId = null
 	pushkey = String(pushkey || '').replace(/ /g, '+')

server/tests/helpers/Schema.test.js

@@ -0,0 +1,243 @@
+require('module-alias/register')
+
+const Schema = require('@helpers/Schema')
+const db = require('@config/database')
+
+afterAll(() => {
+	db.end()
+})
+
+afterEach(() => {
+	Schema.__reset()
+})
+
+describe('verifyScrapeSchema against the real dev schema', () => {
+
+	test('reports ok once the migration has been applied', async () => {
+		const result = await Schema.verifyScrapeSchema()
+
+		if (!result.ok) {
+			throw new Error(
+				`dev scrape schema not applied - run ./migrateDatabase.sh. ` +
+				`missing objects: ${result.missingObjects.join(', ') || 'none'}; ` +
+				`missing privileges: ${result.missingPrivileges.join(', ') || 'none'}`
+			)
+		}
+
+		expect(result.missingObjects).toEqual([])
+		expect(result.missingPrivileges).toEqual([])
+		expect(result.remedy).toBe('')
+	})
+
+	test('isScrapeSchemaReady reflects the last verification', async () => {
+		expect(Schema.isScrapeSchemaReady()).toBe(false)
+
+		await Schema.verifyScrapeSchema()
+		expect(Schema.isScrapeSchemaReady()).toBe(true)
+
+		Schema.__reset()
+		expect(Schema.isScrapeSchemaReady()).toBe(false)
+	})
+
+	test('__setReady is honoured, so request paths can be driven in tests', () => {
+		Schema.__setReady(true)
+		expect(Schema.isScrapeSchemaReady()).toBe(true)
+		Schema.__setReady(false)
+		expect(Schema.isScrapeSchemaReady()).toBe(false)
+	})
+})
+
+//
+// The FULLTEXT index is the one object whose loss is silent: MATCH..AGAINST
+// against a BTREE index returns nothing without erroring. Guard it by type.
+//
+describe('FULLTEXT index guard', () => {
+
+	test('attachment_text_index exists and is actually FULLTEXT', async () => {
+		await expect(Schema.__fulltextIndexExists('attachment', 'attachment_text_index'))
+			.resolves.toBe(true)
+	})
+
+	test('a BTREE index of the same shape would NOT satisfy the check', async () => {
+		// attachment_scrape_state is a real BTREE index, so this proves the
+		// check discriminates on type rather than just on name existing.
+		await expect(Schema.__indexExists('attachment', 'attachment_scrape_state'))
+			.resolves.toBe(true)
+		await expect(Schema.__fulltextIndexExists('attachment', 'attachment_scrape_state'))
+			.resolves.toBe(false)
+	})
+
+	test('verifyScrapeSchema includes it among the verified objects', async () => {
+		const result = await Schema.verifyScrapeSchema()
+		expect(result.ok).toBe(true)
+		expect(result.missingObjects).toEqual([])
+	})
+
+	test('MATCH..AGAINST actually works, not just that an index exists', async () => {
+		// The functional end of the guard. An index of the right type that
+		// somehow could not be searched would still be a broken search.
+		const [rows] = await db.promise().query(
+			`SELECT COUNT(*) AS n FROM attachment
+			 WHERE MATCH(text) AGAINST(? IN NATURAL LANGUAGE MODE)`,
+			['the']
+		)
+		// Any result (including 0 matches) proves the query planner accepted
+		// MATCH..AGAINST against this column, which a BTREE index would reject.
+		expect(typeof rows[0].n).toBe('number')
+	})
+})
+
+describe('object existence checks', () => {
+
+	test('finds the real table, columns and index', async () => {
+		await expect(Schema.__tableExists('scrape_cache')).resolves.toBe(true)
+		await expect(Schema.__columnExists('attachment', 'scrape_state')).resolves.toBe(true)
+		await expect(Schema.__columnExists('attachment', 'scrape_attempts')).resolves.toBe(true)
+		await expect(Schema.__columnExists('attachment', 'enqueued_at')).resolves.toBe(true)
+		await expect(Schema.__indexExists('attachment', 'attachment_scrape_state')).resolves.toBe(true)
+	})
+
+	test('reports absence rather than throwing for things that do not exist', async () => {
+		await expect(Schema.__tableExists('no_such_table_xyz')).resolves.toBe(false)
+		await expect(Schema.__columnExists('attachment', 'no_such_column_xyz')).resolves.toBe(false)
+		await expect(Schema.__indexExists('attachment', 'no_such_index_xyz')).resolves.toBe(false)
+	})
+})
+
+//
+// The privilege check is what turns a future narrow-grants problem into one
+// clear boot message instead of a swallowed runtime failure.
+//
+describe('parseGrants', () => {
+
+	const DB = 'application'
+	const TABLES = ['attachment', 'scrape_cache']
+	const parse = (lines) => Schema.parseGrants(lines, DB, TABLES)
+
+	test('ALL PRIVILEGES ON *.* grants everything', () => {
+		const granted = parse(['GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION'])
+
+		expect(granted.has('SELECT')).toBe(true)
+		expect(granted.has('INSERT')).toBe(true)
+		expect(granted.has('UPDATE')).toBe(true)
+		expect(granted.has('DELETE')).toBe(true)
+	})
+
+	test('ALL PRIVILEGES scoped to the database grants everything', () => {
+		const granted = parse(['GRANT ALL PRIVILEGES ON `application`.* TO `app`@`localhost`'])
+		expect(granted.has('DELETE')).toBe(true)
+	})
+
+	test('reads an explicit privilege list', () => {
+		const granted = parse([
+			'GRANT USAGE ON *.* TO `app`@`localhost`',
+			'GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`',
+		])
+
+		expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE'])
+	})
+
+	// The case that caused this whole change class.
+	test('spots a missing DELETE', () => {
+		const granted = parse(['GRANT SELECT, INSERT, UPDATE ON `application`.* TO `app`@`localhost`'])
+
+		expect(granted.has('UPDATE')).toBe(true)
+		expect(granted.has('DELETE')).toBe(false)
+	})
+
+	test('accepts table-scoped grants for our tables', () => {
+		const granted = parse([
+			'GRANT SELECT, DELETE ON `application`.`attachment` TO `app`@`localhost`',
+			'GRANT INSERT, UPDATE ON `application`.`scrape_cache` TO `app`@`localhost`',
+		])
+
+		expect([...granted].sort()).toEqual(['DELETE', 'INSERT', 'SELECT', 'UPDATE'])
+	})
+
+	test('ignores grants on an unrelated database', () => {
+		const granted = parse(['GRANT ALL PRIVILEGES ON `somethingelse`.* TO `app`@`localhost`'])
+		expect(granted.size).toBe(0)
+	})
+
+	test('ignores USAGE, which conveys no privileges', () => {
+		const granted = parse(['GRANT USAGE ON *.* TO `app`@`localhost`'])
+		expect(granted.size).toBe(0)
+	})
+
+	test('handles grants written without backticks', () => {
+		const granted = parse(['GRANT SELECT, DELETE ON application.* TO app@localhost'])
+		expect(granted.has('DELETE')).toBe(true)
+	})
+
+	test('strips column-level qualifiers', () => {
+		const granted = parse(['GRANT SELECT (id, url), UPDATE (text) ON `application`.`attachment` TO `app`@`localhost`'])
+
+		expect(granted.has('SELECT')).toBe(true)
+		expect(granted.has('UPDATE')).toBe(true)
+	})
+
+	test('ignores lines that are not grant statements', () => {
+		expect(parse(['', 'not a grant line', null]).size).toBe(0)
+	})
+})
+
+//
+// The regression that matters most.
+//
+// Schema.js used to issue CREATE TABLE / ALTER TABLE through the application's
+// own connection pool, which failed on production's least-privilege user with
+// "CREATE command denied to user 'app'@'localhost'". Nothing in this module may
+// ever issue DDL again.
+//
+describe('issues no DDL', () => {
+
+	test('verifyScrapeSchema runs only read-only statements', async () => {
+		const statements = []
+
+		// db.promise() hands back a NEW wrapper on every call, so spying on one
+		// instance would miss the one Schema.js obtains internally. Intercept
+		// the factory and instrument each wrapper it returns.
+		const realPromise = db.promise.bind(db)
+		const spy = jest.spyOn(db, 'promise').mockImplementation(() => {
+			const wrapper = realPromise()
+			const originalQuery = wrapper.query.bind(wrapper)
+
+			wrapper.query = (sql, params) => {
+				statements.push(String(sql))
+				return originalQuery(sql, params)
+			}
+
+			return wrapper
+		})
+
+		await Schema.verifyScrapeSchema()
+
+		spy.mockRestore()
+
+		expect(statements.length).toBeGreaterThan(0)
+
+		const forbidden = /\b(CREATE|ALTER|DROP|TRUNCATE|RENAME)\b/i
+		const offenders = statements.filter(statement => forbidden.test(statement))
+
+		expect(offenders).toEqual([])
+	})
+
+	test('the module source contains no DDL outside comments', () => {
+		const fs = require('fs')
+		const path = require('path')
+
+		const source = fs.readFileSync(
+			path.join(__dirname, '..', '..', 'helpers', 'Schema.js'), 'utf8'
+		)
+
+		const code = source
+			.split('\n')
+			.filter(line => !line.trim().startsWith('//'))
+			.join('\n')
+
+		expect(code).not.toMatch(/\bCREATE\s+TABLE\b/i)
+		expect(code).not.toMatch(/\bALTER\s+TABLE\b/i)
+		expect(code).not.toMatch(/\bCREATE\s+INDEX\b/i)
+		expect(code).not.toMatch(/\bDROP\b/i)
+	})
+})

server/tests/models/Attachment.test.js

@@ -1,6 +1,7 @@
 const Attachment = require('../../models/Attachment')
 const User = require('../../models/User')
 const ScrapeQueue = require('@workers/ScrapeQueue')
+const Schema = require('@helpers/Schema')
 const Robots = require('@helpers/Robots')
 const FetchMock = require('../support/fetchMock')
 const dns = require('dns').promises
@@ -82,6 +83,8 @@ describe('Push URL', () => {
 
 	beforeEach(() => {
 		FetchMock.install()
+		// The server marks this at boot; there is no boot phase in tests.
+		Schema.__setReady(true)
 		Robots.__resetCache()
 		dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
 		// Real code path, no politeness waiting and no robots lookups.
@@ -92,6 +95,7 @@ describe('Push URL', () => {
 		FetchMock.restore()
 		dnsSpy.mockRestore()
 		ScrapeQueue.__reset()
+		Schema.__reset()
 	})
 
 	test('scrapes a pushed url into a searchable attachment', () => {

server/tests/models/ScrapeQueue.test.js

@@ -17,6 +17,7 @@ const testUserName = 'jestTestUserScrapeQueue'
 const password = 'Beans19934!!!'
 
 let userId = null
+let noteId = null
 let dnsSpy = null
 
 const page = (title, description) => FetchMock.html(`
@@ -35,7 +36,11 @@ const attachmentRow = (id) => db.promise()
 	.then(([rows]) => rows[0] || null)
 
 beforeAll(async () => {
-	await Schema.ensureScrapeSchema()
+	// The server normally does this at boot; tests have no boot phase.
+	const verification = await Schema.verifyScrapeSchema()
+	if (!verification.ok) {
+		throw new Error(`dev scrape schema not applied - run ./migrateDatabase.sh (${verification.remedy})`)
+	}
 
 	try {
 		const existing = await User.getByUserName(testUserName)
@@ -44,10 +49,19 @@ beforeAll(async () => {
 
 	const registered = await User.register(testUserName, password)
 	userId = registered.userId
+
+	// A real note, so the soft-delete path below sees an existing note and
+	// hides the attachment rather than deleting it.
+	const [note] = await db.promise().query(
+		'INSERT INTO note (user_id, created, opened) VALUES (?, ?, ?)',
+		[userId, Math.round(Date.now() / 1000), Math.round(Date.now() / 1000)]
+	)
+	noteId = note.insertId
 })
 
 beforeEach(() => {
 	FetchMock.install()
+	Schema.__setReady(true)
 	Robots.__resetCache()
 	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
 	HttpFetch.__setSleep(() => Promise.resolve())
@@ -65,6 +79,7 @@ afterEach(() => {
 
 afterAll(async () => {
 	await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
+	await db.promise().query('DELETE FROM note WHERE user_id = ?', [userId])
 	db.end()
 })
 
@@ -305,6 +320,92 @@ describe('robots.txt', () => {
 	})
 })
 
+//
+// Production ran for a while with the scrape columns missing, and every note
+// save attempted an INSERT that could not succeed. These pin the degraded
+// behaviour: do nothing, quietly, and leave everything else working.
+//
+describe('degrades cleanly when the scrape schema is absent', () => {
+
+	beforeEach(() => {
+		Schema.__setReady(false)
+	})
+
+	test('scanTextForWebsites creates no rows and makes no requests', async () => {
+		FetchMock.route(/.*/, () => page('Should Not Be Fetched'))
+
+		const [before] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
+
+		const text = await Attachment.scanTextForWebsites(userId, null, 'see https://example.com/unavailable')
+
+		const [after] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
+
+		expect(after[0].n).toBe(before[0].n)
+		expect(FetchMock.callCount()).toBe(0)
+		expect(typeof text).toBe('string')
+	})
+
+	//
+	// Removing a link from a note must still clean up its attachment. That path
+	// touches none of the new columns, so it is unaffected by the outage.
+	//
+	// Note the existing semantics: scanTextForWebsites calls
+	// Attachment.delete(..., urlDelete = true), which HARD deletes. The
+	// visible = 0 soft-delete applies only when a user removes an attachment
+	// directly while its note still exists (urlDelete = false).
+	//
+	test('still removes an attachment whose url left the note', async () => {
+		Schema.__setReady(true)
+		const url = 'https://example.com/will-be-removed'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Removed Later'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, noteId, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId, attachmentId, url })
+		expect(await attachmentRow(attachmentId)).not.toBeNull()
+
+		// Now the schema "disappears" and the note is saved without that link.
+		Schema.__setReady(false)
+		await Attachment.scanTextForWebsites(userId, noteId, 'no links here any more')
+
+		// Attachment.delete is deliberately fire-and-forget in scanTextForWebsites,
+		// so give it a tick to land.
+		await new Promise(resolve => setTimeout(resolve, 150))
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+	})
+
+	test('scrapeUrlsCreateAttachments resolves empty without touching the DB', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(Attachment.scrapeUrlsCreateAttachments(userId, null, ['https://example.com/x']))
+			.resolves.toBe('')
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('pushUrl keeps its contract and resolves truthy', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(Attachment.pushUrl('any-key', 'https://example.com/x')).resolves.toBe(true)
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('recoverPending returns zero without querying', async () => {
+		await expect(ScrapeQueue.recoverPending()).resolves.toEqual({ requeued: 0, retired: 0 })
+	})
+
+	test('enqueueUrl is a no-op backstop', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(ScrapeQueue.enqueueUrl({
+			userId, noteId: null, attachmentId: null, url: 'https://example.com/x',
+		})).resolves.toBe('')
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})
+
 describe('crash recovery', () => {
 
 	test('re-enqueues rows left pending by a restart', async () => {

server/tests/schema/migrations.test.js

@@ -0,0 +1,220 @@
+require('module-alias/register')
+
+const { execFileSync } = require('child_process')
+const fs = require('fs')
+const path = require('path')
+
+const REPO = path.join(__dirname, '..', '..', '..')
+
+//
+// These shell out to the Prisma CLI, which only exists inside nix-shell. Skip
+// rather than fail when it is absent, so `npx jest` still works outside the
+// shell -- but do NOT skip silently in a way that hides real drift when it IS
+// available.
+//
+const hasPrisma = (() => {
+	try {
+		execFileSync('prisma', ['--version'], { stdio: 'ignore' })
+		return true
+	} catch (error) {
+		return false
+	}
+})()
+
+const describeWithPrisma = hasPrisma ? describe : describe.skip
+
+if (!hasPrisma) {
+	console.log('[schema tests] prisma not on PATH - skipping. Run inside nix-shell to exercise these.')
+}
+
+const prisma = (args) => execFileSync('prisma', args, {
+	cwd: REPO,
+	encoding: 'utf8',
+	env: process.env,
+})
+
+describe('committed migration artifacts', () => {
+
+	test('the baseline migration and lock file are present', () => {
+		expect(fs.existsSync(path.join(REPO, 'prisma/schema.prisma'))).toBe(true)
+		expect(fs.existsSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'))).toBe(true)
+		expect(fs.existsSync(path.join(REPO, 'prisma/migrations/migration_lock.toml'))).toBe(true)
+	})
+
+	//
+	// Prisma emits COLLATE utf8mb4_unicode_ci for MySQL, but every table in this
+	// database is utf8mb4_general_ci, and `prisma migrate diff` does NOT report
+	// the difference. So a regenerated migration would silently build a
+	// differently-collated database. This asserts the hand-correction is intact.
+	//
+	test('no migration declares a collation other than utf8mb4_general_ci', () => {
+		const dir = path.join(REPO, 'prisma/migrations')
+		const files = fs.readdirSync(dir, { withFileTypes: true })
+			.filter(entry => entry.isDirectory())
+			.map(entry => path.join(dir, entry.name, 'migration.sql'))
+			.filter(file => fs.existsSync(file))
+
+		expect(files.length).toBeGreaterThan(0)
+
+		files.forEach(file => {
+			const ddl = fs.readFileSync(file, 'utf8')
+				.split('\n')
+				.filter(line => !line.trim().startsWith('--'))
+				.join('\n')
+
+			const collations = [...ddl.matchAll(/COLLATE\s+([A-Za-z0-9_]+)/gi)].map(match => match[1])
+			collations.forEach(collation => {
+				expect(collation).toBe('utf8mb4_general_ci')
+			})
+		})
+	})
+
+	test('the baseline creates the FULLTEXT index, not a plain one', () => {
+		const sql = fs.readFileSync(path.join(REPO, 'prisma/migrations/0_init/migration.sql'), 'utf8')
+
+		expect(sql).toMatch(/FULLTEXT INDEX `attachment_text_index`/)
+		// A plain INDEX of that name would be the silent-failure mode.
+		expect(sql).not.toMatch(/^\s*INDEX `attachment_text_index`/m)
+	})
+
+	//
+	// @@fulltext is GA in Prisma 7 -- the `fullTextIndex` preview flag no longer
+	// exists in the CLI at all (verified: zero occurrences in the 7.8.0 bundle).
+	// It was silently ignored rather than rejected, which makes it exactly the
+	// kind of dead config that misleads the next reader.
+	//
+	test('the schema declares @@fulltext and no longer carries the dead preview flag', () => {
+		const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
+
+		expect(schema).toMatch(/@@fulltext\(\[text\]/)
+		expect(schema).not.toMatch(/fullTextIndex/)
+	})
+
+	//
+	// No generator block means `prisma generate` cannot produce a client into
+	// this repo. That is the structural guarantee behind "migrations only" --
+	// stronger than a convention, because it cannot be done by accident.
+	//
+	test('the schema declares no generator, so no client can be generated', () => {
+		const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
+
+		expect(schema).not.toMatch(/generator\s+\w+\s*\{/)
+		expect(schema).toMatch(/datasource\s+db\s*\{/)
+	})
+
+	//
+	// The running server must never depend on Prisma.
+	//
+	test('no application code imports prisma', () => {
+		const { execFileSync } = require('child_process')
+		let hits = ''
+		try {
+			hits = execFileSync('grep', [
+				'-rn', '--include=*.js',
+				'-E', "require\\(['\"]@?prisma", path.join(REPO, 'server'),
+			], { encoding: 'utf8' })
+		} catch (error) {
+			hits = '' // grep exits 1 when nothing matches
+		}
+		expect(hits.trim()).toBe('')
+	})
+
+	test('no @relation is introduced - this database has no foreign keys', () => {
+		// Adding relations would mean adding FK constraints to a live database,
+		// which is a schema change well outside adopting a migration tool.
+		const schema = fs.readFileSync(path.join(REPO, 'prisma/schema.prisma'), 'utf8')
+		expect(schema).not.toMatch(/@relation/)
+	})
+})
+
+describeWithPrisma('schema drift', () => {
+
+	//
+	// The check that keeps schema.prisma honest: if the live database and the
+	// committed schema ever diverge, this fails.
+	//
+	test('schema.prisma matches the live database exactly', () => {
+		const output = prisma([
+			'migrate', 'diff',
+			'--from-config-datasource',
+			'--to-schema', 'prisma/schema.prisma',
+			'--script',
+		])
+
+		expect(output).toMatch(/This is an empty migration/)
+	})
+
+	//
+	// The load-bearing guarantee: the committed migrations, applied to an empty
+	// database, reproduce the live schema exactly.
+	//
+	// Done by actually rebuilding into a scratch database and comparing
+	// information_schema, rather than via `migrate diff --from-migrations`
+	// (which requires a shadow database) -- and this is the stronger check
+	// anyway, because it also catches collation drift, which Prisma's own diff
+	// does NOT report.
+	//
+	test('the committed migrations rebuild the live schema exactly', () => {
+		const db = require('@config/database')
+		const SCRATCH = 'prisma_drift_check'
+
+		const mysqlBin = (() => {
+			try { execFileSync('mariadb', ['--version'], { stdio: 'ignore' }); return 'mariadb' }
+			catch (error) { return 'mysql' }
+		})()
+
+		const adminArgs = ['-u', 'root']
+		const socket = path.join(REPO, '.mysql', 'mysql.sock')
+		if (fs.existsSync(socket)) { adminArgs.push(`--socket=${socket}`) }
+
+		const admin = (sql, database) => execFileSync(
+			mysqlBin,
+			[...adminArgs, '-N', '-B', ...(database ? [database] : []), '-e', sql],
+			{ encoding: 'utf8' }
+		)
+
+		try {
+			admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`; CREATE DATABASE \`${SCRATCH}\`;`)
+
+			execFileSync('prisma', ['migrate', 'deploy'], {
+				cwd: REPO,
+				encoding: 'utf8',
+				env: { ...process.env, DATABASE_URL: `mysql://[email protected]:3306/${SCRATCH}` },
+			})
+
+			const snapshot = (schema) => ({
+				columns: admin(
+					`SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, COLLATION_NAME
+					 FROM information_schema.COLUMNS
+					 WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
+					 ORDER BY TABLE_NAME, COLUMN_NAME`
+				),
+				indexes: admin(
+					`SELECT TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX, COLUMN_NAME, NON_UNIQUE, INDEX_TYPE
+					 FROM information_schema.STATISTICS
+					 WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
+					 ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX`
+				),
+				collations: admin(
+					`SELECT TABLE_NAME, TABLE_COLLATION FROM information_schema.TABLES
+					 WHERE TABLE_SCHEMA='${schema}' AND TABLE_NAME <> '_prisma_migrations'
+					 ORDER BY TABLE_NAME`
+				),
+			})
+
+			const live = snapshot('application')
+			const rebuilt = snapshot(SCRATCH)
+
+			expect(rebuilt.columns).toBe(live.columns)
+			expect(rebuilt.indexes).toBe(live.indexes)
+			expect(rebuilt.collations).toBe(live.collations)
+		} finally {
+			try { admin(`DROP DATABASE IF EXISTS \`${SCRATCH}\`;`) } catch (error) { /* best effort */ }
+		}
+	})
+
+	test('migrate status reports no pending migrations', () => {
+		const output = prisma(['migrate', 'status'])
+		expect(output).toMatch(/Database schema is up to date/)
+	})
+})

server/workers/ScrapeQueue.js

@@ -21,6 +21,7 @@ const HttpFetch = require('@helpers/HttpFetch')
 const HostQueue = require('@helpers/HostQueue')
 const Robots = require('@helpers/Robots')
 const Realtime = require('@helpers/Realtime')
+const Schema = require('@helpers/Schema')
 const SiteScrape = require('@helpers/SiteScrape')
 const ScrapeCache = require('@models/ScrapeCache')
 
@@ -354,6 +355,10 @@ ScrapeQueue.enqueueUrl = ({ userId, noteId, attachmentId, url }) => {
 const runJob = async ({ userId, noteId, attachmentId, url }) => {
 	if (!settings.enabled) { return '' }
 
+	// Backstop, so no future caller can reach the DB layer unguarded when the
+	// scrape columns are absent.
+	if (!Schema.isScrapeSchemaReady()) { return '' }
+
 	const pending = perUser.get(userId) || 0
 	if (pending >= settings.maxPendingPerUser) {
 		// Left at scrape_state = 0, so recoverPending() picks it up later.
@@ -426,6 +431,10 @@ ScrapeQueue.awaitUrl = async (promise, capMs) => {
 ScrapeQueue.recoverPending = async () => {
 	if (!settings.enabled) { return { requeued: 0, retired: 0 } }
 
+	// Every query below references scrape_state, which only exists once an
+	// admin has run ./migrateDatabase.sh.
+	if (!Schema.isScrapeSchemaReady()) { return { requeued: 0, retired: 0 } }
+
 	// Anything left at FETCHING died with the previous process.
 	await db.promise().query(
 		'UPDATE attachment SET scrape_state = ? WHERE scrape_state = ?',

shell.nix

@@ -11,6 +11,17 @@ pkgs.mkShell {
     pm2
     mariadb
     openssl
+
+    # Schema migrations and DB scaffolding only -- the running server never
+    # imports Prisma. The nixpkgs wrapper is self-configuring: it exports
+    # PRISMA_SCHEMA_ENGINE_BINARY pointing at a nix-built prisma-engines and
+    # execs the bundled CLI, so there is no engine download and none of the
+    # usual NixOS prebuilt-binary trouble.
+    #
+    # Pinned to prisma_7 deliberately, NOT the `prisma` alias: Prisma 8 has no
+    # MariaDB support yet, so if nixpkgs moves that alias to v8 this project's
+    # database would stop being supported underneath us.
+    prisma_7
   ];
 
   shellHook = ''
@@ -79,8 +90,21 @@ pkgs.mkShell {
       done
       echo "MySQL ready."
 
+      #
+      # Collation is explicit because the server default here is
+      # utf8mb4_unicode_ci while every application table is
+      # utf8mb4_general_ci. Without this, a table created later without an
+      # explicit COLLATE would silently inherit the wrong one, changing string
+      # comparison and FULLTEXT ranking. Prisma does not model collation and
+      # `prisma migrate diff` does not report the difference.
+      #
+      # Existing databases need a one-time, non-destructive fix:
+      #   ALTER DATABASE application
+      #     CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+      #
       mysql -u root --socket="$MYSQL_SOCKET" \
-        -e "CREATE DATABASE IF NOT EXISTS application;" 2>/dev/null
+        -e "CREATE DATABASE IF NOT EXISTS application \
+              CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" 2>/dev/null
     fi
 
     #
@@ -98,6 +122,24 @@ pkgs.mkShell {
       export DB_USER=root
       export DB_PASS=
 
+      #
+      # Prisma CLI connection string, for migrations and scaffolding.
+      #
+      # The warning above about shadowing ~/.env does NOT apply here:
+      # DATABASE_URL is read only by the Prisma CLI, never by
+      # server/config/database.js, so it cannot affect the running server.
+      #
+      # Deliberately NOT derived from DB_USER/DB_PASS. Prisma needs DDL
+      # rights, and in production those variables hold the least-privilege
+      # `app` account. Admin credentials are composed by migrateDatabase.sh
+      # instead, which is why this export lives only in the dev branch.
+      #
+      export DATABASE_URL="mysql://root@127.0.0.1:3306/application"
+
+      # Stop the Prisma CLI phoning checkpoint.prisma.io for update checks.
+      # Keeps the schema tests network-free and the output uncluttered.
+      export CHECKPOINT_DISABLE=1
+
       # Stable dev JWT key, persisted across shell sessions
       if [ -z "''${JSON_KEY:-}" ]; then
         if [ -s "$PWD/.mysql/jwt_key" ]; then