// // Idempotent schema convergence for the scraper tables. // // There is no migration framework in this project and no schema file in git, // so this runs at boot, checks information_schema for each object, and applies // only what is missing. Dev and prod self-heal, and deploy.sh needs no change. // // Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8. // ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used - // every ALTER is guarded by an explicit existence check instead. // const db = require('@config/database') let Schema = module.exports = {} const SCRAPE_CACHE_TABLE = ` CREATE TABLE IF NOT EXISTS scrape_cache ( id int(10) unsigned NOT NULL AUTO_INCREMENT, url_hash char(64) NOT NULL, url varchar(1000) NOT NULL DEFAULT '', canonical_url varchar(1000) DEFAULT NULL, status smallint(6) NOT NULL DEFAULT 0, title varchar(512) DEFAULT NULL, site_name varchar(255) DEFAULT NULL, description mediumtext DEFAULT NULL, keywords varchar(512) DEFAULT NULL, search_text mediumtext DEFAULT NULL, image_url varchar(1000) DEFAULT NULL, file_location varchar(500) DEFAULT NULL, etag varchar(255) DEFAULT NULL, last_modified varchar(128) DEFAULT NULL, content_type varchar(128) DEFAULT NULL, fetch_state varchar(20) NOT NULL DEFAULT 'pending', failure_count int(11) NOT NULL DEFAULT 0, last_error varchar(255) DEFAULT NULL, fetched_at int(11) DEFAULT NULL, expires_at int(11) DEFAULT NULL, PRIMARY KEY (id), UNIQUE KEY scrape_cache_url_hash (url_hash), KEY scrape_cache_expires_at (expires_at), KEY scrape_cache_fetch_state (fetch_state) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ` // DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep. const ATTACHMENT_COLUMNS = [ { name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' }, { name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' }, { name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' }, ] const ATTACHMENT_INDEXES = [ { name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' }, ] const columnExists = async (table, column) => { const [rows] = await db.promise().query( `SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`, [table, column] ) return rows.length > 0 } const indexExists = async (table, index) => { const [rows] = await db.promise().query( `SELECT 1 FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`, [table, index] ) return rows.length > 0 } Schema.__columnExists = columnExists Schema.__indexExists = indexExists // // Safe to call repeatedly. Returns the list of statements it actually applied, // so boot logging says something useful on the one run that matters. // Schema.ensureScrapeSchema = async () => { const applied = [] await db.promise().query(SCRAPE_CACHE_TABLE) for (const column of ATTACHMENT_COLUMNS) { if (await columnExists('attachment', column.name)) { continue } await db.promise().query(column.ddl) applied.push(`attachment.${column.name}`) } // Index last: it references the columns above. for (const index of ATTACHMENT_INDEXES) { if (await indexExists('attachment', index.name)) { continue } await db.promise().query(index.ddl) applied.push(`attachment KEY ${index.name}`) } if (applied.length) { console.log('[schema] applied scrape schema changes:', applied.join(', ')) } return applied }