Deploy and schema updates for testing
This commit is contained in:
Max committed
1 parent
8d60aae65c
commit
81b4162117
8 files changed
+389
-86
No files matched your search
+167
-61
@@ -1,58 +1,57 @@
|
||||
//
|
||||
// Idempotent schema convergence for the scraper tables.
|
||||
// Verification of the scraper's schema and privileges. Issues NO DDL.
|
||||
//
|
||||
// There is no migration framework in this project and no schema file in git,
|
||||
// so this runs at boot, checks information_schema for each object, and applies
|
||||
// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
|
||||
// This module used to converge the schema at boot with CREATE TABLE / ALTER
|
||||
// TABLE. That worked only because dev connects as root (shell.nix exports
|
||||
// DB_USER=root); on production, which runs as a least-privilege `app` user, it
|
||||
// failed with:
|
||||
//
|
||||
// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
|
||||
// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
|
||||
// every ALTER is guarded by an explicit existence check instead.
|
||||
// CREATE command denied to user 'app'@'localhost' for table scrape_cache
|
||||
//
|
||||
// The restricted account is correct - the server ingests untrusted URLs and
|
||||
// should not be able to reshape its own tables. So DDL moved out of the
|
||||
// application entirely: an admin runs ./applyScrapeSchema.sh once, and this
|
||||
// module only checks that the result is present and usable.
|
||||
//
|
||||
// Note on why the DDL is REMOVED rather than guarded by an existence check:
|
||||
// MariaDB evaluates the CREATE privilege before IF NOT EXISTS, so
|
||||
// `CREATE TABLE IF NOT EXISTS` would keep erroring for a DML-only user on every
|
||||
// boot even once the table exists.
|
||||
//
|
||||
|
||||
const db = require('@config/database')
|
||||
|
||||
let Schema = module.exports = {}
|
||||
|
||||
const SCRAPE_CACHE_TABLE = `
|
||||
CREATE TABLE IF NOT EXISTS scrape_cache (
|
||||
id int(10) unsigned NOT NULL AUTO_INCREMENT,
|
||||
url_hash char(64) NOT NULL,
|
||||
url varchar(1000) NOT NULL DEFAULT '',
|
||||
canonical_url varchar(1000) DEFAULT NULL,
|
||||
status smallint(6) NOT NULL DEFAULT 0,
|
||||
title varchar(512) DEFAULT NULL,
|
||||
site_name varchar(255) DEFAULT NULL,
|
||||
description mediumtext DEFAULT NULL,
|
||||
keywords varchar(512) DEFAULT NULL,
|
||||
search_text mediumtext DEFAULT NULL,
|
||||
image_url varchar(1000) DEFAULT NULL,
|
||||
file_location varchar(500) DEFAULT NULL,
|
||||
etag varchar(255) DEFAULT NULL,
|
||||
last_modified varchar(128) DEFAULT NULL,
|
||||
content_type varchar(128) DEFAULT NULL,
|
||||
fetch_state varchar(20) NOT NULL DEFAULT 'pending',
|
||||
failure_count int(11) NOT NULL DEFAULT 0,
|
||||
last_error varchar(255) DEFAULT NULL,
|
||||
fetched_at int(11) DEFAULT NULL,
|
||||
expires_at int(11) DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY scrape_cache_url_hash (url_hash),
|
||||
KEY scrape_cache_expires_at (expires_at),
|
||||
KEY scrape_cache_fetch_state (fetch_state)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
`
|
||||
const REQUIRED_TABLE = 'scrape_cache'
|
||||
const REQUIRED_COLUMNS = ['scrape_state', 'scrape_attempts', 'enqueued_at']
|
||||
const REQUIRED_INDEX = 'attachment_scrape_state'
|
||||
|
||||
// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
|
||||
const ATTACHMENT_COLUMNS = [
|
||||
{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
|
||||
{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
|
||||
{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
|
||||
]
|
||||
// The scraper writes and deletes, not just reads. DELETE is genuinely needed:
|
||||
// terminal failures drop the shell attachment row (ScrapeQueue), and
|
||||
// ScrapeCache.purgeExpired removes expired rows.
|
||||
const REQUIRED_PRIVILEGES = ['SELECT', 'INSERT', 'UPDATE', 'DELETE']
|
||||
|
||||
const ATTACHMENT_INDEXES = [
|
||||
{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
|
||||
]
|
||||
// Set by verifyScrapeSchema so the per-request paths can check a flag rather
|
||||
// than querying information_schema on every note save.
|
||||
let ready = false
|
||||
let lastResult = null
|
||||
|
||||
Schema.isScrapeSchemaReady = () => ready
|
||||
Schema.lastVerification = () => lastResult
|
||||
|
||||
// Test seam.
|
||||
Schema.__setReady = (value) => { ready = !!value }
|
||||
Schema.__reset = () => { ready = false; lastResult = null }
|
||||
|
||||
const tableExists = async (table) => {
|
||||
const [rows] = await db.promise().query(
|
||||
`SELECT 1 FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? LIMIT 1`,
|
||||
[table]
|
||||
)
|
||||
return rows.length > 0
|
||||
}
|
||||
|
||||
const columnExists = async (table, column) => {
|
||||
const [rows] = await db.promise().query(
|
||||
@@ -71,34 +70,141 @@ const indexExists = async (table, index) => {
|
||||
)
|
||||
return rows.length > 0
|
||||
}
|
||||
|
||||
Schema.__tableExists = tableExists
|
||||
Schema.__columnExists = columnExists
|
||||
Schema.__indexExists = indexExists
|
||||
|
||||
//
|
||||
// Safe to call repeatedly. Returns the list of statements it actually applied,
|
||||
// so boot logging says something useful on the one run that matters.
|
||||
// Parse SHOW GRANTS output into the set of privileges that apply to this
|
||||
// database. Any user may run SHOW GRANTS FOR CURRENT_USER(), so this needs no
|
||||
// privileges of its own.
|
||||
//
|
||||
Schema.ensureScrapeSchema = async () => {
|
||||
const applied = []
|
||||
// Grant lines look like:
|
||||
// GRANT ALL PRIVILEGES ON *.* TO `root`@`localhost` WITH GRANT OPTION
|
||||
// GRANT SELECT, INSERT, UPDATE, DELETE ON `application`.* TO `app`@`localhost`
|
||||
// GRANT SELECT ON `application`.`attachment` TO `app`@`localhost`
|
||||
// GRANT USAGE ON *.* TO `app`@`localhost`
|
||||
//
|
||||
const PRIVILEGE_PATTERN = /^GRANT\s+(.+?)\s+ON\s+(\S+)\s+TO\s/i
|
||||
|
||||
await db.promise().query(SCRAPE_CACHE_TABLE)
|
||||
Schema.parseGrants = (lines, database, tables) => {
|
||||
const granted = new Set()
|
||||
|
||||
for (const column of ATTACHMENT_COLUMNS) {
|
||||
if (await columnExists('attachment', column.name)) { continue }
|
||||
await db.promise().query(column.ddl)
|
||||
applied.push(`attachment.${column.name}`)
|
||||
// Scope strings that cover our tables. Backticks are optional in output.
|
||||
const strip = (value) => value.replace(/`/g, '')
|
||||
const relevantScopes = new Set([
|
||||
'*.*',
|
||||
`${database}.*`,
|
||||
...tables.map(table => `${database}.${table}`),
|
||||
])
|
||||
|
||||
for (const line of lines) {
|
||||
const match = String(line).match(PRIVILEGE_PATTERN)
|
||||
if (!match) { continue }
|
||||
|
||||
const scope = strip(match[2])
|
||||
if (!relevantScopes.has(scope)) { continue }
|
||||
|
||||
// Drop any column-level qualifiers, e.g. "SELECT (id, url)".
|
||||
const privileges = match[1].replace(/\s*\([^)]*\)/g, '')
|
||||
|
||||
for (const privilege of privileges.split(',')) {
|
||||
const name = privilege.trim().toUpperCase()
|
||||
if (!name || name === 'USAGE') { continue }
|
||||
|
||||
if (name === 'ALL PRIVILEGES' || name === 'ALL') {
|
||||
REQUIRED_PRIVILEGES.forEach(required => granted.add(required))
|
||||
continue
|
||||
}
|
||||
|
||||
granted.add(name)
|
||||
}
|
||||
}
|
||||
|
||||
// Index last: it references the columns above.
|
||||
for (const index of ATTACHMENT_INDEXES) {
|
||||
if (await indexExists('attachment', index.name)) { continue }
|
||||
await db.promise().query(index.ddl)
|
||||
applied.push(`attachment KEY ${index.name}`)
|
||||
return granted
|
||||
}
|
||||
|
||||
const checkPrivileges = async () => {
|
||||
let lines = []
|
||||
|
||||
try {
|
||||
const [rows] = await db.promise().query('SHOW GRANTS FOR CURRENT_USER()')
|
||||
// Each row is an object with a single, server-named column.
|
||||
lines = rows.map(row => Object.values(row)[0])
|
||||
} catch (error) {
|
||||
// If we cannot read our own grants, do not block on a guess - the
|
||||
// object checks above are the real gate.
|
||||
console.log('[schema] could not read grants, skipping privilege check:', error.message)
|
||||
return { missing: [], database: null, currentUser: null }
|
||||
}
|
||||
|
||||
const [dbRows] = await db.promise().query('SELECT DATABASE() AS name, CURRENT_USER() AS who')
|
||||
const database = dbRows[0] && dbRows[0].name
|
||||
const currentUser = dbRows[0] && dbRows[0].who
|
||||
|
||||
const granted = Schema.parseGrants(lines, database, ['attachment', REQUIRED_TABLE])
|
||||
|
||||
return {
|
||||
missing: REQUIRED_PRIVILEGES.filter(privilege => !granted.has(privilege)),
|
||||
database,
|
||||
currentUser,
|
||||
}
|
||||
}
|
||||
|
||||
// 'app@localhost' -> "'app'@'localhost'", so the remedy is copy-pasteable.
|
||||
const quoteUser = (currentUser) => {
|
||||
if (!currentUser) { return '<app user>' }
|
||||
|
||||
const [name, host] = String(currentUser).split('@')
|
||||
return `'${name}'@'${host || 'localhost'}'`
|
||||
}
|
||||
|
||||
//
|
||||
// Check that everything the scraper needs is present. Never mutates anything.
|
||||
//
|
||||
// Returns { ok, missingObjects, missingPrivileges, remedy }.
|
||||
//
|
||||
Schema.verifyScrapeSchema = async () => {
|
||||
const missingObjects = []
|
||||
|
||||
if (!await tableExists(REQUIRED_TABLE)) {
|
||||
missingObjects.push(`table ${REQUIRED_TABLE}`)
|
||||
}
|
||||
|
||||
for (const column of REQUIRED_COLUMNS) {
|
||||
if (!await columnExists('attachment', column)) {
|
||||
missingObjects.push(`attachment.${column}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (!await indexExists('attachment', REQUIRED_INDEX)) {
|
||||
missingObjects.push(`attachment index ${REQUIRED_INDEX}`)
|
||||
}
|
||||
|
||||
const privileges = await checkPrivileges()
|
||||
const missingPrivileges = privileges.missing || []
|
||||
|
||||
const remedy = []
|
||||
if (missingObjects.length) {
|
||||
remedy.push('run ./applyScrapeSchema.sh as a database admin')
|
||||
}
|
||||
if (missingPrivileges.length) {
|
||||
// Name the real database and user so this can be pasted straight into
|
||||
// a mysql prompt rather than decoded first.
|
||||
const target = `${privileges.database || '<database>'}.*`
|
||||
remedy.push(
|
||||
`GRANT ${missingPrivileges.join(', ')} ON ${target} TO ${quoteUser(privileges.currentUser)}`
|
||||
)
|
||||
}
|
||||
|
||||
if (applied.length) {
|
||||
console.log('[schema] applied scrape schema changes:', applied.join(', '))
|
||||
lastResult = {
|
||||
ok: missingObjects.length === 0 && missingPrivileges.length === 0,
|
||||
missingObjects,
|
||||
missingPrivileges,
|
||||
remedy: remedy.join('; '),
|
||||
}
|
||||
ready = lastResult.ok
|
||||
|
||||
return applied
|
||||
return lastResult
|
||||
}
|
||||
Reference in new issue
Block a user