Testing Migrations for web scraper

migrateDatabase.sh

@@ -156,6 +156,35 @@ HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES
 
 if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then
 	echo "Existing schema found with no migration history -- baselining."
+
+	#
+	# Report drift BEFORE recording the baseline.
+	#
+	# `migrate resolve` records a migration without executing it, so baselining
+	# a database that does not actually match the schema leaves it permanently
+	# short of those objects while Prisma reports everything as applied. That
+	# is exactly how production ended up missing scrape_cache: this script
+	# previously inferred "the attachment table exists" => "the whole schema
+	# exists", which is not the same claim.
+	#
+	# Later migrations are additive and idempotent, so drift is recoverable --
+	# but the operator should see it rather than discover it from a log line
+	# days later.
+	#
+	if ! prisma migrate diff \
+		--from-config-datasource \
+		--to-schema prisma/schema.prisma \
+		--exit-code >/dev/null 2>&1
+	then
+		echo ""
+		echo "NOTE: this database does not yet match prisma/schema.prisma."
+		echo "      Baselining 0_init anyway, then applying later migrations to"
+		echo "      converge it. Differences:"
+		prisma migrate diff --from-config-datasource --to-schema prisma/schema.prisma \
+			| sed 's/^/        /'
+		echo ""
+	fi
+
 	echo "Marking 0_init as already applied (NOT executing it)."
 	prisma migrate resolve --applied 0_init
 	echo ""
@@ -207,14 +236,52 @@ echo ""
 		AND TABLE_COLLATION <> 'utf8mb4_general_ci';
 "
 
-# The FULLTEXT index backs Attachment.textSearch. If a migration ever converts
+#
-# it to a BTREE index, attachment search silently returns nothing -- so fail
+# Assert that everything the application actually needs now exists.
-# loudly here rather than letting that ship.
+#
+# This is the check that was missing. `prisma migrate deploy` reporting success
+# only means the recorded migrations ran -- it says nothing about whether the
+# database ended up in the intended shape, which is precisely how production
+# could pass this script and still log
+# "Scraper: DISABLED - missing table scrape_cache ...".
+#
+# Mirrors server/helpers/Schema.js's boot verification, so a green run here
+# means a green boot there.
+#
+FAILURES=0
+require_object() {
+	local label="$1" count="$2"
+	if [ "${count:-0}" -eq 0 ]; then
+		echo "FAILED: missing $label" >&2
+		FAILURES=$((FAILURES + 1))
+	fi
+}
+
+require_object "table scrape_cache" "$(sql "SELECT COUNT(*) FROM information_schema.TABLES
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='scrape_cache'")"
+
+for column in scrape_state scrape_attempts enqueued_at; do
+	require_object "attachment.$column" "$(sql "SELECT COUNT(*) FROM information_schema.COLUMNS
+		WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND COLUMN_NAME='$column'")"
+done
+
+require_object "attachment index attachment_scrape_state" "$(sql "SELECT COUNT(*) FROM information_schema.STATISTICS
+	WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND INDEX_NAME='attachment_scrape_state'")"
+
+# Checked by TYPE, not just by name: a BTREE index called attachment_text_index
+# would make MATCH..AGAINST return nothing at all, with no error anywhere.
 FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS
 	WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'")
 if [ "$FT" != "FULLTEXT" ]; then
 	echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2
-	echo "Attachment search depends on MATCH..AGAINST and is now broken." >&2
+	echo "Attachment search depends on MATCH..AGAINST and would silently return nothing." >&2
+	FAILURES=$((FAILURES + 1))
+fi
+
+if [ "$FAILURES" -gt 0 ]; then
+	echo "" >&2
+	echo "$FAILURES required object(s) missing after migrating -- the scraper will stay disabled." >&2
+	echo "This should not happen; check 'prisma migrate status' and the migration SQL." >&2
 	exit 1
 fi
 

prisma/migrations/0002_scrape_objects_repair/migration.sql

@@ -0,0 +1,74 @@
+--
+-- Repair: converge environments that were baselined before the scrape objects
+-- existed.
+--
+-- WHY THIS EXISTS
+--
+-- The 0_init baseline was generated from a database that already had
+-- scrape_cache and the attachment.scrape_* columns, on the understanding that
+-- every environment was in that state. Production was not. Because
+-- `prisma migrate resolve --applied 0_init` records a migration WITHOUT
+-- executing it, production ended up with 0_init marked applied while those
+-- objects were never created -- so the scraper logged:
+--
+--   Scraper: DISABLED - missing table scrape_cache, attachment.scrape_state, ...
+--
+-- Re-running 0_init is not an option (it would try to CREATE TABLE over ~2200
+-- rows of live data), so this additive, idempotent migration brings any
+-- environment up to the intended schema instead.
+--
+-- Every statement uses IF NOT EXISTS, so this is a clean no-op on environments
+-- that already had these objects. MariaDB 11.4 supports that syntax for columns
+-- and indexes; both dev and production are MariaDB, so this is safe here. It
+-- would NOT be valid on MySQL 8.
+--
+-- The definitions below are identical to those in 0_init -- deliberately so,
+-- because the end state must be the same whichever path an environment took.
+--
+
+-- One row per scraped URL, shared across users and notes.
+--
+-- url_hash carries the UNIQUE index rather than `url` because a UNIQUE on
+-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit.
+--
+-- Collation is explicit: Prisma renders utf8mb4_unicode_ci, but every table in
+-- this database is utf8mb4_general_ci, and `prisma migrate diff` does not
+-- report the difference.
+CREATE TABLE IF NOT EXISTS `scrape_cache` (
+    `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
+    `url_hash` CHAR(64) NOT NULL,
+    `url` VARCHAR(1000) NOT NULL DEFAULT '',
+    `canonical_url` VARCHAR(1000) NULL,
+    `status` SMALLINT NOT NULL DEFAULT 0,
+    `title` VARCHAR(512) NULL,
+    `site_name` VARCHAR(255) NULL,
+    `description` MEDIUMTEXT NULL,
+    `keywords` VARCHAR(512) NULL,
+    `search_text` MEDIUMTEXT NULL,
+    `image_url` VARCHAR(1000) NULL,
+    `file_location` VARCHAR(500) NULL,
+    `etag` VARCHAR(255) NULL,
+    `last_modified` VARCHAR(128) NULL,
+    `content_type` VARCHAR(128) NULL,
+    `fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
+    `failure_count` INTEGER NOT NULL DEFAULT 0,
+    `last_error` VARCHAR(255) NULL,
+    `fetched_at` INTEGER NULL,
+    `expires_at` INTEGER NULL,
+
+    UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
+    INDEX `scrape_cache_expires_at`(`expires_at`),
+    INDEX `scrape_cache_fetch_state`(`fetch_state`),
+    PRIMARY KEY (`id`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+
+-- Per-attachment scrape lifecycle, so the in-memory queue survives a restart.
+--
+-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
+-- DEFAULT 2 is load-bearing -- it keeps every pre-existing attachment row
+-- (~2200 in production) out of the scraper's boot recovery sweep.
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state`    TINYINT NOT NULL DEFAULT 2;
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` TINYINT NOT NULL DEFAULT 0;
+ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at`     INTEGER NULL;
+
+CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);