75 lines
3.3 KiB
SQL
75 lines
3.3 KiB
SQL
--
|
|
-- Repair: converge environments that were baselined before the scrape objects
|
|
-- existed.
|
|
--
|
|
-- WHY THIS EXISTS
|
|
--
|
|
-- The 0_init baseline was generated from a database that already had
|
|
-- scrape_cache and the attachment.scrape_* columns, on the understanding that
|
|
-- every environment was in that state. Production was not. Because
|
|
-- `prisma migrate resolve --applied 0_init` records a migration WITHOUT
|
|
-- executing it, production ended up with 0_init marked applied while those
|
|
-- objects were never created -- so the scraper logged:
|
|
--
|
|
-- Scraper: DISABLED - missing table scrape_cache, attachment.scrape_state, ...
|
|
--
|
|
-- Re-running 0_init is not an option (it would try to CREATE TABLE over ~2200
|
|
-- rows of live data), so this additive, idempotent migration brings any
|
|
-- environment up to the intended schema instead.
|
|
--
|
|
-- Every statement uses IF NOT EXISTS, so this is a clean no-op on environments
|
|
-- that already had these objects. MariaDB 11.4 supports that syntax for columns
|
|
-- and indexes; both dev and production are MariaDB, so this is safe here. It
|
|
-- would NOT be valid on MySQL 8.
|
|
--
|
|
-- The definitions below are identical to those in 0_init -- deliberately so,
|
|
-- because the end state must be the same whichever path an environment took.
|
|
--
|
|
|
|
-- One row per scraped URL, shared across users and notes.
|
|
--
|
|
-- url_hash carries the UNIQUE index rather than `url` because a UNIQUE on
|
|
-- varchar(1000) utf8mb4 is 4000 bytes and exceeds InnoDB's 3072-byte key limit.
|
|
--
|
|
-- Collation is explicit: Prisma renders utf8mb4_unicode_ci, but every table in
|
|
-- this database is utf8mb4_general_ci, and `prisma migrate diff` does not
|
|
-- report the difference.
|
|
CREATE TABLE IF NOT EXISTS `scrape_cache` (
|
|
`id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`url_hash` CHAR(64) NOT NULL,
|
|
`url` VARCHAR(1000) NOT NULL DEFAULT '',
|
|
`canonical_url` VARCHAR(1000) NULL,
|
|
`status` SMALLINT NOT NULL DEFAULT 0,
|
|
`title` VARCHAR(512) NULL,
|
|
`site_name` VARCHAR(255) NULL,
|
|
`description` MEDIUMTEXT NULL,
|
|
`keywords` VARCHAR(512) NULL,
|
|
`search_text` MEDIUMTEXT NULL,
|
|
`image_url` VARCHAR(1000) NULL,
|
|
`file_location` VARCHAR(500) NULL,
|
|
`etag` VARCHAR(255) NULL,
|
|
`last_modified` VARCHAR(128) NULL,
|
|
`content_type` VARCHAR(128) NULL,
|
|
`fetch_state` VARCHAR(20) NOT NULL DEFAULT 'pending',
|
|
`failure_count` INTEGER NOT NULL DEFAULT 0,
|
|
`last_error` VARCHAR(255) NULL,
|
|
`fetched_at` INTEGER NULL,
|
|
`expires_at` INTEGER NULL,
|
|
|
|
UNIQUE INDEX `scrape_cache_url_hash`(`url_hash`),
|
|
INDEX `scrape_cache_expires_at`(`expires_at`),
|
|
INDEX `scrape_cache_fetch_state`(`fetch_state`),
|
|
PRIMARY KEY (`id`)
|
|
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
|
|
|
|
-- Per-attachment scrape lifecycle, so the in-memory queue survives a restart.
|
|
--
|
|
-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
|
|
-- DEFAULT 2 is load-bearing -- it keeps every pre-existing attachment row
|
|
-- (~2200 in production) out of the scraper's boot recovery sweep.
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state` TINYINT NOT NULL DEFAULT 2;
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` TINYINT NOT NULL DEFAULT 0;
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at` INTEGER NULL;
|
|
|
|
CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);
|