68 lines
3.0 KiB
SQL
68 lines
3.0 KiB
SQL
--
|
|
-- Scrape cache + per-attachment scrape lifecycle.
|
|
--
|
|
-- HOW THIS IS APPLIED
|
|
--
|
|
-- ./applyScrapeSchema.sh (from the repo root, as an admin)
|
|
--
|
|
-- Run it once per environment. The application does NOT apply this itself:
|
|
-- production connects as a least-privilege `app` user with DML rights only, and
|
|
-- deliberately holds no CREATE/ALTER. server/helpers/Schema.js only *verifies*
|
|
-- that these objects exist and that the required privileges are held; if they
|
|
-- are missing it disables link previews and logs the remedy.
|
|
--
|
|
-- Safe to re-run: every statement is conditional.
|
|
--
|
|
-- Portability: dev and production are both MariaDB 11.4, so ADD COLUMN IF NOT
|
|
-- EXISTS and CREATE INDEX IF NOT EXISTS are available. If this project ever
|
|
-- moves to MySQL 8 these become invalid and need information_schema guards
|
|
-- instead.
|
|
--
|
|
|
|
-- One row per URL, shared across users and notes.
|
|
--
|
|
-- url_hash carries the UNIQUE index because a UNIQUE on varchar(1000) utf8mb4
|
|
-- is 4000 bytes and exceeds InnoDB's 3072-byte key limit. `url` is kept
|
|
-- human-readable for debugging only.
|
|
--
|
|
-- COLLATE is stated explicitly to match the existing tables (utf8mb4_general_ci)
|
|
-- rather than inheriting whatever the server default happens to be.
|
|
CREATE TABLE IF NOT EXISTS `scrape_cache` (
|
|
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
|
|
`url_hash` char(64) NOT NULL,
|
|
`url` varchar(1000) NOT NULL DEFAULT '',
|
|
`canonical_url` varchar(1000) DEFAULT NULL,
|
|
`status` smallint(6) NOT NULL DEFAULT 0,
|
|
`title` varchar(512) DEFAULT NULL,
|
|
`site_name` varchar(255) DEFAULT NULL,
|
|
`description` mediumtext DEFAULT NULL,
|
|
`keywords` varchar(512) DEFAULT NULL,
|
|
`search_text` mediumtext DEFAULT NULL,
|
|
`image_url` varchar(1000) DEFAULT NULL,
|
|
`file_location` varchar(500) DEFAULT NULL,
|
|
`etag` varchar(255) DEFAULT NULL,
|
|
`last_modified` varchar(128) DEFAULT NULL,
|
|
`content_type` varchar(128) DEFAULT NULL,
|
|
`fetch_state` varchar(20) NOT NULL DEFAULT 'pending',
|
|
`failure_count` int(11) NOT NULL DEFAULT 0,
|
|
`last_error` varchar(255) DEFAULT NULL,
|
|
`fetched_at` int(11) DEFAULT NULL,
|
|
`expires_at` int(11) DEFAULT NULL,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
|
|
KEY `scrape_cache_expires_at` (`expires_at`),
|
|
KEY `scrape_cache_fetch_state` (`fetch_state`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
|
|
|
|
-- Per-attachment lifecycle, so the in-memory scrape queue survives a restart.
|
|
--
|
|
-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
|
|
--
|
|
-- DEFAULT 2 is load-bearing: it keeps every pre-existing attachment row
|
|
-- (~2200 in production) out of the boot recovery sweep.
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_state` tinyint(4) NOT NULL DEFAULT 2;
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
|
|
ALTER TABLE `attachment` ADD COLUMN IF NOT EXISTS `enqueued_at` int(11) DEFAULT NULL;
|
|
|
|
CREATE INDEX IF NOT EXISTS `attachment_scrape_state` ON `attachment` (`scrape_state`, `enqueued_at`);
|