Adding scrape files for testing

server/helpers/Schema.js

@@ -0,0 +1,104 @@
+//
+// Idempotent schema convergence for the scraper tables.
+//
+// There is no migration framework in this project and no schema file in git,
+// so this runs at boot, checks information_schema for each object, and applies
+// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
+//
+// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
+// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
+// every ALTER is guarded by an explicit existence check instead.
+//
+
+const db = require('@config/database')
+
+let Schema = module.exports = {}
+
+const SCRAPE_CACHE_TABLE = `
+	CREATE TABLE IF NOT EXISTS scrape_cache (
+		id            int(10) unsigned NOT NULL AUTO_INCREMENT,
+		url_hash      char(64) NOT NULL,
+		url           varchar(1000) NOT NULL DEFAULT '',
+		canonical_url varchar(1000) DEFAULT NULL,
+		status        smallint(6) NOT NULL DEFAULT 0,
+		title         varchar(512) DEFAULT NULL,
+		site_name     varchar(255) DEFAULT NULL,
+		description   mediumtext DEFAULT NULL,
+		keywords      varchar(512) DEFAULT NULL,
+		search_text   mediumtext DEFAULT NULL,
+		image_url     varchar(1000) DEFAULT NULL,
+		file_location varchar(500) DEFAULT NULL,
+		etag          varchar(255) DEFAULT NULL,
+		last_modified varchar(128) DEFAULT NULL,
+		content_type  varchar(128) DEFAULT NULL,
+		fetch_state   varchar(20) NOT NULL DEFAULT 'pending',
+		failure_count int(11) NOT NULL DEFAULT 0,
+		last_error    varchar(255) DEFAULT NULL,
+		fetched_at    int(11) DEFAULT NULL,
+		expires_at    int(11) DEFAULT NULL,
+		PRIMARY KEY (id),
+		UNIQUE KEY scrape_cache_url_hash (url_hash),
+		KEY scrape_cache_expires_at (expires_at),
+		KEY scrape_cache_fetch_state (fetch_state)
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+`
+
+// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
+const ATTACHMENT_COLUMNS = [
+	{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
+	{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
+	{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
+]
+
+const ATTACHMENT_INDEXES = [
+	{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
+]
+
+const columnExists = async (table, column) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.COLUMNS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
+		[table, column]
+	)
+	return rows.length > 0
+}
+
+const indexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+Schema.__columnExists = columnExists
+Schema.__indexExists = indexExists
+
+//
+// Safe to call repeatedly. Returns the list of statements it actually applied,
+// so boot logging says something useful on the one run that matters.
+//
+Schema.ensureScrapeSchema = async () => {
+	const applied = []
+
+	await db.promise().query(SCRAPE_CACHE_TABLE)
+
+	for (const column of ATTACHMENT_COLUMNS) {
+		if (await columnExists('attachment', column.name)) { continue }
+		await db.promise().query(column.ddl)
+		applied.push(`attachment.${column.name}`)
+	}
+
+	// Index last: it references the columns above.
+	for (const index of ATTACHMENT_INDEXES) {
+		if (await indexExists('attachment', index.name)) { continue }
+		await db.promise().query(index.ddl)
+		applied.push(`attachment KEY ${index.name}`)
+	}
+
+	if (applied.length) {
+		console.log('[schema] applied scrape schema changes:', applied.join(', '))
+	}
+
+	return applied
+}