52 lines
2.3 KiB
SQL
52 lines
2.3 KiB
SQL
--
|
|
-- Scrape cache + per-attachment scrape lifecycle.
|
|
--
|
|
-- This file is the human-readable record. It is NOT applied by hand: the same
|
|
-- statements live in server/helpers/Schema.js and are applied idempotently at
|
|
-- boot after an information_schema existence check, so dev and prod converge
|
|
-- without a migration runner.
|
|
--
|
|
-- Portability: dev is MariaDB 11.4, prod is MySQL 8. Nothing here may use
|
|
-- ADD COLUMN IF NOT EXISTS (MariaDB-only).
|
|
--
|
|
|
|
-- One row per URL, shared across users and notes. url_hash carries the UNIQUE
|
|
-- index because a UNIQUE on varchar(1000) utf8mb4 is 4000 bytes and exceeds
|
|
-- InnoDB's 3072-byte key limit.
|
|
CREATE TABLE IF NOT EXISTS `scrape_cache` (
|
|
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
|
|
`url_hash` char(64) NOT NULL,
|
|
`url` varchar(1000) NOT NULL DEFAULT '',
|
|
`canonical_url` varchar(1000) DEFAULT NULL,
|
|
`status` smallint(6) NOT NULL DEFAULT 0,
|
|
`title` varchar(512) DEFAULT NULL,
|
|
`site_name` varchar(255) DEFAULT NULL,
|
|
`description` mediumtext DEFAULT NULL,
|
|
`keywords` varchar(512) DEFAULT NULL,
|
|
`search_text` mediumtext DEFAULT NULL,
|
|
`image_url` varchar(1000) DEFAULT NULL,
|
|
`file_location` varchar(500) DEFAULT NULL,
|
|
`etag` varchar(255) DEFAULT NULL,
|
|
`last_modified` varchar(128) DEFAULT NULL,
|
|
`content_type` varchar(128) DEFAULT NULL,
|
|
`fetch_state` varchar(20) NOT NULL DEFAULT 'pending',
|
|
`failure_count` int(11) NOT NULL DEFAULT 0,
|
|
`last_error` varchar(255) DEFAULT NULL,
|
|
`fetched_at` int(11) DEFAULT NULL,
|
|
`expires_at` int(11) DEFAULT NULL,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
|
|
KEY `scrape_cache_expires_at` (`expires_at`),
|
|
KEY `scrape_cache_fetch_state` (`fetch_state`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
|
|
-- Per-attachment lifecycle, so an in-memory queue survives a restart.
|
|
--
|
|
-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
|
|
-- DEFAULT 2 is load-bearing: it keeps the ~2200 pre-existing rows out of the
|
|
-- recovery sweep on first boot.
|
|
ALTER TABLE `attachment` ADD COLUMN `scrape_state` tinyint(4) NOT NULL DEFAULT 2;
|
|
ALTER TABLE `attachment` ADD COLUMN `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
|
|
ALTER TABLE `attachment` ADD COLUMN `enqueued_at` int(11) DEFAULT NULL;
|
|
ALTER TABLE `attachment` ADD KEY `attachment_scrape_state` (`scrape_state`, `enqueued_at`);
|