diff --git a/migrateDatabase.sh b/migrateDatabase.sh index c7f9c71..b4ec32d 100755 --- a/migrateDatabase.sh +++ b/migrateDatabase.sh @@ -156,6 +156,35 @@ HAS_EXISTING_SCHEMA=$(sql "SELECT COUNT(*) FROM information_schema.TABLES if [ "${HAS_MIGRATION_TABLE:-0}" -eq 0 ] && [ "${HAS_EXISTING_SCHEMA:-0}" -gt 0 ]; then echo "Existing schema found with no migration history -- baselining." + + # + # Report drift BEFORE recording the baseline. + # + # `migrate resolve` records a migration without executing it, so baselining + # a database that does not actually match the schema leaves it permanently + # short of those objects while Prisma reports everything as applied. That + # is exactly how production ended up missing scrape_cache: this script + # previously inferred "the attachment table exists" => "the whole schema + # exists", which is not the same claim. + # + # Later migrations are additive and idempotent, so drift is recoverable -- + # but the operator should see it rather than discover it from a log line + # days later. + # + if ! prisma migrate diff \ + --from-config-datasource \ + --to-schema prisma/schema.prisma \ + --exit-code >/dev/null 2>&1 + then + echo "" + echo "NOTE: this database does not yet match prisma/schema.prisma." + echo " Baselining 0_init anyway, then applying later migrations to" + echo " converge it. Differences:" + prisma migrate diff --from-config-datasource --to-schema prisma/schema.prisma \ + | sed 's/^/ /' + echo "" + fi + echo "Marking 0_init as already applied (NOT executing it)." prisma migrate resolve --applied 0_init echo "" @@ -207,14 +236,52 @@ echo "" AND TABLE_COLLATION <> 'utf8mb4_general_ci'; " -# The FULLTEXT index backs Attachment.textSearch. If a migration ever converts -# it to a BTREE index, attachment search silently returns nothing -- so fail -# loudly here rather than letting that ship. +# +# Assert that everything the application actually needs now exists. +# +# This is the check that was missing. `prisma migrate deploy` reporting success +# only means the recorded migrations ran -- it says nothing about whether the +# database ended up in the intended shape, which is precisely how production +# could pass this script and still log +# "Scraper: DISABLED - missing table scrape_cache ...". +# +# Mirrors server/helpers/Schema.js's boot verification, so a green run here +# means a green boot there. +# +FAILURES=0 +require_object() { + local label="$1" count="$2" + if [ "${count:-0}" -eq 0 ]; then + echo "FAILED: missing $label" >&2 + FAILURES=$((FAILURES + 1)) + fi +} + +require_object "table scrape_cache" "$(sql "SELECT COUNT(*) FROM information_schema.TABLES + WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='scrape_cache'")" + +for column in scrape_state scrape_attempts enqueued_at; do + require_object "attachment.$column" "$(sql "SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND COLUMN_NAME='$column'")" +done + +require_object "attachment index attachment_scrape_state" "$(sql "SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA='$DB_NAME' AND TABLE_NAME='attachment' AND INDEX_NAME='attachment_scrape_state'")" + +# Checked by TYPE, not just by name: a BTREE index called attachment_text_index +# would make MATCH..AGAINST return nothing at all, with no error anywhere. FT=$(sql "SELECT IFNULL(MAX(INDEX_TYPE),'MISSING') FROM information_schema.STATISTICS WHERE TABLE_SCHEMA='$DB_NAME' AND INDEX_NAME='attachment_text_index'") if [ "$FT" != "FULLTEXT" ]; then echo "FAILED: attachment_text_index is '$FT', expected FULLTEXT." >&2 - echo "Attachment search depends on MATCH..AGAINST and is now broken." >&2 + echo "Attachment search depends on MATCH..AGAINST and would silently return nothing." >&2 + FAILURES=$((FAILURES + 1)) +fi + +if [ "$FAILURES" -gt 0 ]; then + echo "" >&2 + echo "$FAILURES required object(s) missing after migrating -- the scraper will stay disabled." >&2 + echo "This should not happen; check 'prisma migrate status' and the migration SQL." >&2 exit 1 fi diff --git a/prisma/migrations/0002_scrape_objects_repair/migration.sql b/prisma/migrations/0002_scrape_objects_repair/migration.sql new file mode 100644 index 0000000..7dea770 --- /dev/null +++ b/prisma/migrations/0002_scrape_objects_repair/migration.sql @@ -0,0 +1,74 @@ +-- +-- Repair: converge environments that were baselined before the scrape objects +-- existed. +-- +-- WHY THIS EXISTS +-- +-- The 0_init baseline was generated from a database that already had +-- scrape_cache and the attachment.scrape_* columns, on the understanding that +-- every environment was in that state. Production was not. Because +-- `prisma migrate resolve --applied 0_init` records a migration WITHOUT +-- executing it, production ended up with 0_init marked applied while those +-- objects were never created -- so the scraper logged: +-- +-- Scraper: DISABLED - missing table scrape_cache, attachment.scrape_state, ... +-- +-- Re-running 0_init is not an option (it would try to CREATE TABLE over ~2200 +-- rows of live data), so this additive, idempotent migration brings any +-- environment up to the intended schema instead. +-- +-- Every statement uses IF NOT EXISTS, so this is a clean no-op on environments +-- that already had these objects. MariaDB 11.4 supports that syntax for columns +-- and indexes; both dev and production are MariaDB, so this is safe here. It +-- would NOT be valid on MySQL 8. +-- +-- The definitions below are identical to those in 0_init -- deliberately so, +-- because the end state must be the same whichever path an environment took. +-- + +-- One row per scraped URL, shared across users and notes. +-- +-- url_hash carries the UNIQUE index rather than `url` because a UNIQUE on +-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit. +-- +-- Collation is explicit: Prisma renders utf8mb4_unicode_ci, but every table in +-- this database is utf8mb4_general_ci, and `prisma migrate diff` does not +-- report the difference. +CREATE TABLE IF NOT EXISTS `scrape_cache` ( + `id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + `url_hash` CHAR(64) NOT NULL, + `url` VARCHAR(1000) NOT NULL DEFAULT '', + `canonical_url` VARCHAR(1000) NULL, + `status` SMALLINT NOT NULL DEFAULT 0, + `title` VARCHAR(512) NULL, + `site_name` VARCHAR(255) NULL, + `description` MEDIUMTEXT NULL, + `keywords` VARCHAR(512) NULL, + `search_text` MEDIUMTEXT NULL, + `image_url` VARCHAR(1000) NULL, + `file_location` VARCHAR(500) NULL, + `etag` VARCHAR(255) NULL, + `last_modified` VARCHAR(128) NULL, + `content_type` VARCHAR(128) NULL, + `fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending', + `failure_count` INTEGER NOT NULL DEFAULT 0, + `last_error` VARCHAR(255) NULL, + `fetched_at` INTEGER NULL, + `expires_at` INTEGER NULL, + + UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`), + INDEX `scrape_cache_expires_at`(`expires_at`), + INDEX `scrape_cache_fetch_state`(`fetch_state`), + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; + +-- Per-attachment scrape lifecycle, so the in-memory queue survives a restart. +-- +-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed. +-- DEFAULT 2 is load-bearing -- it keeps every pre-existing attachment row +-- (~2200 in production) out of the scraper's boot recovery sweep. +ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state` TINYINT NOT NULL DEFAULT 2; +ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` TINYINT NOT NULL DEFAULT 0; +ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at` INTEGER NULL; + +CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);