105 lines
3.7 KiB
JavaScript
105 lines
3.7 KiB
JavaScript
//
|
|
// Idempotent schema convergence for the scraper tables.
|
|
//
|
|
// There is no migration framework in this project and no schema file in git,
|
|
// so this runs at boot, checks information_schema for each object, and applies
|
|
// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
|
|
//
|
|
// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
|
|
// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
|
|
// every ALTER is guarded by an explicit existence check instead.
|
|
//
|
|
|
|
const db = require('@config/database')
|
|
|
|
let Schema = module.exports = {}
|
|
|
|
const SCRAPE_CACHE_TABLE = `
|
|
CREATE TABLE IF NOT EXISTS scrape_cache (
|
|
id int(10) unsigned NOT NULL AUTO_INCREMENT,
|
|
url_hash char(64) NOT NULL,
|
|
url varchar(1000) NOT NULL DEFAULT '',
|
|
canonical_url varchar(1000) DEFAULT NULL,
|
|
status smallint(6) NOT NULL DEFAULT 0,
|
|
title varchar(512) DEFAULT NULL,
|
|
site_name varchar(255) DEFAULT NULL,
|
|
description mediumtext DEFAULT NULL,
|
|
keywords varchar(512) DEFAULT NULL,
|
|
search_text mediumtext DEFAULT NULL,
|
|
image_url varchar(1000) DEFAULT NULL,
|
|
file_location varchar(500) DEFAULT NULL,
|
|
etag varchar(255) DEFAULT NULL,
|
|
last_modified varchar(128) DEFAULT NULL,
|
|
content_type varchar(128) DEFAULT NULL,
|
|
fetch_state varchar(20) NOT NULL DEFAULT 'pending',
|
|
failure_count int(11) NOT NULL DEFAULT 0,
|
|
last_error varchar(255) DEFAULT NULL,
|
|
fetched_at int(11) DEFAULT NULL,
|
|
expires_at int(11) DEFAULT NULL,
|
|
PRIMARY KEY (id),
|
|
UNIQUE KEY scrape_cache_url_hash (url_hash),
|
|
KEY scrape_cache_expires_at (expires_at),
|
|
KEY scrape_cache_fetch_state (fetch_state)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
`
|
|
|
|
// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
|
|
const ATTACHMENT_COLUMNS = [
|
|
{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
|
|
{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
|
|
{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
|
|
]
|
|
|
|
const ATTACHMENT_INDEXES = [
|
|
{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
|
|
]
|
|
|
|
const columnExists = async (table, column) => {
|
|
const [rows] = await db.promise().query(
|
|
`SELECT 1 FROM information_schema.COLUMNS
|
|
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
|
|
[table, column]
|
|
)
|
|
return rows.length > 0
|
|
}
|
|
|
|
const indexExists = async (table, index) => {
|
|
const [rows] = await db.promise().query(
|
|
`SELECT 1 FROM information_schema.STATISTICS
|
|
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
|
|
[table, index]
|
|
)
|
|
return rows.length > 0
|
|
}
|
|
Schema.__columnExists = columnExists
|
|
Schema.__indexExists = indexExists
|
|
|
|
//
|
|
// Safe to call repeatedly. Returns the list of statements it actually applied,
|
|
// so boot logging says something useful on the one run that matters.
|
|
//
|
|
Schema.ensureScrapeSchema = async () => {
|
|
const applied = []
|
|
|
|
await db.promise().query(SCRAPE_CACHE_TABLE)
|
|
|
|
for (const column of ATTACHMENT_COLUMNS) {
|
|
if (await columnExists('attachment', column.name)) { continue }
|
|
await db.promise().query(column.ddl)
|
|
applied.push(`attachment.${column.name}`)
|
|
}
|
|
|
|
// Index last: it references the columns above.
|
|
for (const index of ATTACHMENT_INDEXES) {
|
|
if (await indexExists('attachment', index.name)) { continue }
|
|
await db.promise().query(index.ddl)
|
|
applied.push(`attachment KEY ${index.name}`)
|
|
}
|
|
|
|
if (applied.length) {
|
|
console.log('[schema] applied scrape schema changes:', applied.join(', '))
|
|
}
|
|
|
|
return applied
|
|
}
|