Adding scrape files for testing

2026-11-03 11:20:03 -07:00
1 parent 7d255fe687
commit 8d60aae65c
@@ -0,0 +1,51 @@
+--
+-- Scrape cache + per-attachment scrape lifecycle.
+--
+-- This file is the human-readable record. It is NOT applied by hand: the same
+-- statements live in server/helpers/Schema.js and are applied idempotently at
+-- boot after an information_schema existence check, so dev and prod converge
+-- without a migration runner.
+--
+-- Portability: dev is MariaDB 11.4, prod is MySQL 8. Nothing here may use
+-- ADD COLUMN IF NOT EXISTS (MariaDB-only).
+--
+
+-- One row per URL, shared across users and notes. url_hash carries the UNIQUE
+-- index because a UNIQUE on varchar(1000) utf8mb4 is 4000 bytes and exceeds
+-- InnoDB's 3072-byte key limit.
+CREATE TABLE IF NOT EXISTS `scrape_cache` (
+  `id`            int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `url_hash`      char(64) NOT NULL,
+  `url`           varchar(1000) NOT NULL DEFAULT '',
+  `canonical_url` varchar(1000) DEFAULT NULL,
+  `status`        smallint(6) NOT NULL DEFAULT 0,
+  `title`         varchar(512) DEFAULT NULL,
+  `site_name`     varchar(255) DEFAULT NULL,
+  `description`   mediumtext DEFAULT NULL,
+  `keywords`      varchar(512) DEFAULT NULL,
+  `search_text`   mediumtext DEFAULT NULL,
+  `image_url`     varchar(1000) DEFAULT NULL,
+  `file_location` varchar(500) DEFAULT NULL,
+  `etag`          varchar(255) DEFAULT NULL,
+  `last_modified` varchar(128) DEFAULT NULL,
+  `content_type`  varchar(128) DEFAULT NULL,
+  `fetch_state`   varchar(20) NOT NULL DEFAULT 'pending',
+  `failure_count` int(11) NOT NULL DEFAULT 0,
+  `last_error`    varchar(255) DEFAULT NULL,
+  `fetched_at`    int(11) DEFAULT NULL,
+  `expires_at`    int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
+  KEY `scrape_cache_expires_at` (`expires_at`),
+  KEY `scrape_cache_fetch_state` (`fetch_state`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Per-attachment lifecycle, so an in-memory queue survives a restart.
+--
+-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
+-- DEFAULT 2 is load-bearing: it keeps the ~2200 pre-existing rows out of the
+-- recovery sweep on first boot.
+ALTER TABLE `attachment` ADD COLUMN `scrape_state`    tinyint(4) NOT NULL DEFAULT 2;
+ALTER TABLE `attachment` ADD COLUMN `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
+ALTER TABLE `attachment` ADD COLUMN `enqueued_at`     int(11) DEFAULT NULL;
+ALTER TABLE `attachment` ADD KEY `attachment_scrape_state` (`scrape_state`, `enqueued_at`);