Adding scrape files for testing

server/config/scrape.js

@@ -0,0 +1,70 @@
+//
+// All scraper tunables in one place, so nothing downstream is a magic number.
+//
+// Env overrides exist for the two kill switches only. Everything else is a
+// constant on purpose: the politeness numbers are the difference between
+// looking like a browser and looking like a bot, and they should not drift
+// per-environment without a code change.
+//
+
+const isTest = process.env.NODE_ENV === 'test'
+
+const config = {
+	// Kill switches
+	enabled: process.env.SCRAPE_ENABLED !== '0',
+	// The background worker is off under Jest so tests drive the queue
+	// deterministically via ScrapeQueue.drain() instead of racing it.
+	workerEnabled: process.env.SCRAPE_WORKER ? process.env.SCRAPE_WORKER === '1' : !isTest,
+
+	// Timeouts
+	pageTimeoutMs: 8000,     // single HTTP request
+	totalBudgetMs: 25000,    // all hops + retries for one URL
+	imageTimeoutMs: 10000,
+
+	// Size caps. Content-Length is optional and can lie, so these are also
+	// enforced while streaming the body.
+	maxHtmlBytes: 2 * 1024 * 1024,
+	maxImageBytes: 5 * 1024 * 1024,
+	maxRobotsBytes: 100 * 1024,
+
+	// Redirects
+	maxRedirects: 5,
+
+	// Retries. Only for transient classes - a 404 is an answer, not a failure.
+	retries: 2,
+	backoffBaseMs: 500,
+	backoffMaxMs: 8000,
+	maxRetryAfterMs: 60 * 1000,
+
+	// Politeness. One request at a time per host, spaced out, is what actually
+	// stops us looking like a crawler.
+	maxConcurrentFetches: 4,
+	perHostConcurrency: 1,
+	perHostMinSpacingMs: 1000,
+	perHostMaxSpacingMs: 3000,
+	maxCrawlDelayMs: 30 * 1000,
+
+	// Backpressure
+	maxQueueLength: 500,
+	maxPendingPerUser: 50,
+
+	// Identity. Stable salt, so a host always sees the same browser across
+	// restarts. Changing this re-rolls every host's assigned profile.
+	profileSalt: 'ss-scrape-v1',
+	robotsUaToken: 'SolidScribeBot',
+	robotsTtlMs: 24 * 60 * 60 * 1000,
+	robotsNegativeTtlMs: 60 * 60 * 1000,
+	robotsErrorTtlMs: 10 * 60 * 1000,
+	robotsMaxOrigins: 2000,
+
+	// Parsing / storage
+	maxSearchTextBytes: 8000,
+	maxKeywordSourceChars: 6000,
+	maxUrlLength: 1000, // matches attachment.url varchar(1000)
+
+	// staticFiles is served from our own origin via /api/static, so an SVG
+	// with an inline script would be same-origin. Off by default.
+	allowSvgThumbnails: false,
+}
+
+module.exports = config

server/helpers/BrowserProfiles.js

@@ -0,0 +1,195 @@
+//
+// Browser identities for outbound fetches.
+//
+// The old code sent a single hardcoded Googlebot User-Agent. That was removed
+// deliberately: Googlebot is verifiable by reverse DNS, so the claim is trivially
+// disproven, and sites that check it block the source IP outright. Some also
+// serve crawlers a stripped-down page.
+//
+// What replaces it is an honest, *coherent* browser identity. Coherence is the
+// point - a Chrome User-Agent shipping Firefox's Accept header, or sec-ch-ua on
+// a Safari request, is a stronger bot signal than any UA string. So each profile
+// carries the complete header set its family really sends, and nothing mixes.
+//
+// Profiles are assigned stably per host, not randomly per request: a site should
+// see one returning browser, not a rotating cast.
+//
+
+const config = require('@config/scrape')
+
+let BrowserProfiles = module.exports = {}
+
+const CHROME_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
+const FIREFOX_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+
+const PROFILES = BrowserProfiles.PROFILES = [
+	{
+		id: 'chrome-win',
+		family: 'chrome',
+		ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+		accept: CHROME_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+		secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
+		secChUaMobile: '?0',
+		secChUaPlatform: '"Windows"',
+	},
+	{
+		id: 'chrome-mac',
+		family: 'chrome',
+		ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+		accept: CHROME_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+		secChUa: '"Chromium";v="141", "Not;A=Brand";v="24", "Google Chrome";v="141"',
+		secChUaMobile: '?0',
+		secChUaPlatform: '"macOS"',
+	},
+	{
+		id: 'firefox-win',
+		family: 'firefox',
+		ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+		accept: FIREFOX_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.5',
+	},
+	{
+		id: 'firefox-linux',
+		family: 'firefox',
+		ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
+		accept: FIREFOX_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.5',
+	},
+	{
+		id: 'safari-mac',
+		family: 'safari',
+		ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
+		accept: SAFARI_ACCEPT,
+		acceptLanguage: 'en-US,en;q=0.9',
+	},
+]
+
+// FNV-1a. Not cryptographic - we only need a stable, well-spread bucket.
+const fnv1a = (input) => {
+	let hash = 0x811c9dc5
+	for (let i = 0; i < input.length; i++) {
+		hash ^= input.charCodeAt(i)
+		hash = Math.imul(hash, 0x01000193) >>> 0
+	}
+	return hash >>> 0
+}
+BrowserProfiles.__fnv1a = fnv1a
+
+//
+// Same host always gets the same profile, across requests and across restarts
+// (the salt is a constant in config, not a per-boot random).
+//
+BrowserProfiles.profileForHost = (hostname) => {
+	const key = require('@helpers/UrlSafety').registrableHost(hostname) + config.profileSalt
+	return PROFILES[fnv1a(key) % PROFILES.length]
+}
+
+BrowserProfiles.profileById = (id) => PROFILES.find(profile => profile.id === id) || PROFILES[0]
+
+//
+// Sec-Fetch-Site is relative: "none" for a URL typed into the bar, "same-origin"
+// when the referrer shares the origin, "cross-site" otherwise. Getting this
+// wrong while sending a Referer is exactly the kind of incoherence that flags.
+//
+const fetchSite = (targetUrl, referer) => {
+	if (!referer) { return 'none' }
+	try {
+		const target = new URL(targetUrl)
+		const from = new URL(referer)
+		if (target.origin === from.origin) { return 'same-origin' }
+		if (UrlSafetyHost(target) === UrlSafetyHost(from)) { return 'same-site' }
+		return 'cross-site'
+	} catch (error) {
+		return 'none'
+	}
+}
+
+const UrlSafetyHost = (url) => require('@helpers/UrlSafety').registrableHost(url.hostname)
+
+const applyChromiumHints = (headers, profile) => {
+	if (profile.family !== 'chrome') { return headers }
+	headers['sec-ch-ua'] = profile.secChUa
+	headers['sec-ch-ua-mobile'] = profile.secChUaMobile
+	headers['sec-ch-ua-platform'] = profile.secChUaPlatform
+	return headers
+}
+
+//
+// Headers for a top-level document navigation.
+//
+BrowserProfiles.documentHeaders = (profile, { url, referer } = {}) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': profile.accept,
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	// Safari sends neither sec-ch-ua nor Sec-Fetch-*; Firefox sends Sec-Fetch-*
+	// but no client hints; Chromium sends both.
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'document'
+		headers['Sec-Fetch-Mode'] = 'navigate'
+		headers['Sec-Fetch-Site'] = fetchSite(url, referer)
+		headers['Sec-Fetch-User'] = '?1'
+		headers['Upgrade-Insecure-Requests'] = '1'
+	}
+
+	applyChromiumHints(headers, profile)
+
+	if (referer) { headers['Referer'] = referer }
+
+	return headers
+}
+
+//
+// Headers for a subresource image load, as issued from the page it was found on.
+//
+BrowserProfiles.imageHeaders = (profile, { url, pageUrl } = {}) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': profile.family === 'chrome'
+			? 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'
+			: 'image/avif,image/webp,*/*',
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'image'
+		headers['Sec-Fetch-Mode'] = 'no-cors'
+		headers['Sec-Fetch-Site'] = fetchSite(url, pageUrl)
+	}
+
+	applyChromiumHints(headers, profile)
+
+	if (pageUrl) { headers['Referer'] = pageUrl }
+
+	return headers
+}
+
+//
+// Headers for robots.txt. Same identity as the page fetch, but it is not a
+// navigation - a browser would never send Sec-Fetch-Dest: document for this.
+//
+BrowserProfiles.robotsHeaders = (profile) => {
+	const headers = {
+		'User-Agent': profile.ua,
+		'Accept': 'text/plain,*/*;q=0.8',
+		'Accept-Language': profile.acceptLanguage,
+		'Accept-Encoding': 'gzip, deflate, br',
+	}
+
+	if (profile.family !== 'safari') {
+		headers['Sec-Fetch-Dest'] = 'empty'
+		headers['Sec-Fetch-Mode'] = 'no-cors'
+		headers['Sec-Fetch-Site'] = 'none'
+	}
+
+	applyChromiumHints(headers, profile)
+
+	return headers
+}

server/helpers/HostQueue.js

@@ -0,0 +1,221 @@
+//
+// Per-host politeness scheduler. Knows nothing about scraping.
+//
+// This is the piece that stops the scraper looking like a crawler. The old code
+// did foundUrls.forEach(fetch) - a note with 40 links opened 40 simultaneous
+// connections, which is the most bot-like traffic shape available. Here, each
+// host gets one request at a time with jittered spacing between them, and a
+// global cap bounds total concurrency.
+//
+
+const config = require('@config/scrape')
+
+let HostQueue = module.exports = {}
+
+const jitter = (min, max) => Math.round(min + Math.random() * Math.max(0, max - min))
+
+class Queue {
+	constructor(options = {}) {
+		this.globalConcurrency = options.globalConcurrency != null
+			? options.globalConcurrency : config.maxConcurrentFetches
+		this.perHostConcurrency = options.perHostConcurrency != null
+			? options.perHostConcurrency : config.perHostConcurrency
+		this.minSpacingMs = options.minSpacingMs != null
+			? options.minSpacingMs : config.perHostMinSpacingMs
+		this.maxSpacingMs = options.maxSpacingMs != null
+			? options.maxSpacingMs : config.perHostMaxSpacingMs
+		this.now = options.now || (() => Date.now())
+		// Injectable so tests with a controlled clock don't spend real
+		// wall-time waiting out politeness spacing.
+		this.setTimeoutFn = options.setTimeoutFn || ((fn, ms) => setTimeout(fn, ms))
+		this.clearTimeoutFn = options.clearTimeoutFn || ((handle) => clearTimeout(handle))
+
+		this.pending = []            // FIFO
+		this.hosts = new Map()       // hostKey -> { running, nextAllowedAt }
+		this.running = 0
+		this.seq = 0
+		this.timer = null
+		this.stopped = false
+		this.drainWaiters = []
+	}
+
+	hostState(hostKey) {
+		if (!this.hosts.has(hostKey)) {
+			this.hosts.set(hostKey, { running: 0, nextAllowedAt: 0 })
+		}
+		return this.hosts.get(hostKey)
+	}
+
+	//
+	// Queue a task for a host. Resolves/rejects with the task's own outcome.
+	// spacingMs overrides the jitter floor (robots.txt Crawl-delay).
+	//
+	schedule(hostKey, taskFn, options = {}) {
+		if (this.stopped) {
+			return Promise.reject(new Error('HostQueue stopped'))
+		}
+
+		return new Promise((resolve, reject) => {
+			this.pending.push({
+				hostKey: hostKey || '(none)',
+				taskFn,
+				resolve,
+				reject,
+				spacingMs: options.spacingMs != null ? options.spacingMs : null,
+				seq: this.seq++,
+			})
+			this.pump()
+		})
+	}
+
+	//
+	// Run whatever is eligible. FIFO scan rather than per-host round-robin, so
+	// no host can starve another and order stays predictable.
+	//
+	pump() {
+		if (this.stopped) { return }
+
+		this.clearTimer()
+
+		while (this.running < this.globalConcurrency) {
+			const at = this.now()
+			let index = -1
+			let earliest = Infinity
+
+			for (let i = 0; i < this.pending.length; i++) {
+				const job = this.pending[i]
+				const host = this.hostState(job.hostKey)
+
+				if (host.running >= this.perHostConcurrency) { continue }
+
+				if (host.nextAllowedAt > at) {
+					earliest = Math.min(earliest, host.nextAllowedAt)
+					continue
+				}
+
+				index = i
+				break
+			}
+
+			if (index === -1) {
+				// Nothing runnable now. Wake up when the earliest host is due.
+				if (this.pending.length && earliest !== Infinity) {
+					this.armTimer(earliest - at)
+				}
+				break
+			}
+
+			this.run(this.pending.splice(index, 1)[0])
+		}
+
+		this.checkDrained()
+	}
+
+	run(job) {
+		const host = this.hostState(job.hostKey)
+		host.running++
+		this.running++
+
+		const settle = () => {
+			host.running--
+			this.running--
+
+			// Space the NEXT request to this host from the end of this one.
+			const spacing = job.spacingMs != null
+				? job.spacingMs
+				: jitter(this.minSpacingMs, this.maxSpacingMs)
+			host.nextAllowedAt = this.now() + spacing
+
+			this.pump()
+		}
+
+		// A throwing taskFn must not wedge the pump, so the call itself is
+		// wrapped rather than only its promise.
+		let result = null
+		try {
+			result = Promise.resolve(job.taskFn())
+		} catch (error) {
+			result = Promise.reject(error)
+		}
+
+		result.then(
+			value => { settle(); job.resolve(value) },
+			error => { settle(); job.reject(error) }
+		)
+	}
+
+	armTimer(delay) {
+		this.clearTimer()
+		this.timer = this.setTimeoutFn(() => {
+			this.timer = null
+			this.pump()
+		}, Math.max(1, delay))
+
+		//
+		// Deliberately NOT unref'd.
+		//
+		// This timer is the only thing that will resume a host which is inside
+		// its politeness gap. unref'ing it means that if nothing else holds the
+		// event loop open, Node exits and every queued job is silently
+		// abandoned with its promise never settling. Inside the server that is
+		// masked by the listening socket, but it makes the queue unusable from
+		// a standalone script and is a footgun either way.
+		//
+		// Shutdown is handled explicitly instead: stop() clears this timer and
+		// rejects whatever is still queued, and it is wired to SIGINT/SIGTERM
+		// in server/index.js.
+		//
+	}
+
+	clearTimer() {
+		if (this.timer) {
+			this.clearTimeoutFn(this.timer)
+			this.timer = null
+		}
+	}
+
+	size() { return this.pending.length }
+	inflight() { return this.running }
+
+	stats() {
+		return {
+			pending: this.pending.length,
+			running: this.running,
+			hosts: this.hosts.size,
+		}
+	}
+
+	checkDrained() {
+		if (this.pending.length || this.running) { return }
+
+		const waiters = this.drainWaiters
+		this.drainWaiters = []
+		waiters.forEach(resolve => resolve())
+	}
+
+	// Resolves once the queue is empty and nothing is in flight.
+	drain() {
+		if (!this.pending.length && !this.running) { return Promise.resolve() }
+		return new Promise(resolve => this.drainWaiters.push(resolve))
+	}
+
+	//
+	// Reject everything still queued. In-flight tasks are not cancelled here -
+	// their own AbortControllers handle that.
+	//
+	stop() {
+		this.stopped = true
+		this.clearTimer()
+
+		const pending = this.pending
+		this.pending = []
+		pending.forEach(job => job.reject(new Error('HostQueue stopped')))
+
+		this.checkDrained()
+	}
+}
+
+HostQueue.Queue = Queue
+
+// Shared instance used by the scrape worker. Tests build their own.
+HostQueue.shared = new Queue()

server/helpers/HttpFetch.js

@@ -0,0 +1,429 @@
+//
+// The only module in the server that opens an outbound connection.
+//
+// Everything that made the old scraper unreliable is handled here, in one place:
+// a single AbortController-based timeout (the old code raced request's socket
+// timeout against a manual setTimeout that could fire against a null reference
+// and crash the process), manual redirect following so the SSRF gate runs on
+// every hop, hard size caps enforced while streaming, a Content-Type gate before
+// any body is read, real charset decoding, and retries that only fire for
+// transient failures.
+//
+
+const config = require('@config/scrape')
+const UrlSafety = require('@helpers/UrlSafety')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+
+let HttpFetch = module.exports = {}
+
+const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']
+
+class FetchError extends Error {
+	constructor(kind, message, extra = {}) {
+		super(message)
+		this.name = 'FetchError'
+		this.kind = kind          // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
+		this.status = extra.status || null
+		this.retryAfterMs = extra.retryAfterMs || null
+		this.url = extra.url || null
+		this.reason = extra.reason || null
+	}
+}
+HttpFetch.FetchError = FetchError
+
+// Injectable so tests don't spend real wall-time in backoff. Far simpler than
+// interleaving fake timers with promises.
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+HttpFetch.__setSleep = (fn) => { sleep = fn }
+HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }
+
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])
+
+const isRetryable = (error) => {
+	if (!(error instanceof FetchError)) { return false }
+	if (error.kind === 'network' || error.kind === 'timeout') { return true }
+	if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
+	return false
+}
+
+// Retry-After is either delta-seconds or an HTTP-date.
+const parseRetryAfter = (value) => {
+	if (!value) { return null }
+
+	const seconds = Number(value)
+	if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }
+
+	const when = Date.parse(value)
+	if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }
+
+	return null
+}
+
+const backoffFor = (attempt) => {
+	const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
+	return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
+}
+
+const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
+const mimeOf = (contentType) => contentType.split(';')[0].trim()
+
+const charsetFromContentType = (contentType) => {
+	const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
+	return match ? match[1].toLowerCase() : null
+}
+
+// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
+// byte maps to a character and the regex can't miss.
+const charsetFromMeta = (buffer) => {
+	const head = buffer.subarray(0, 2048).toString('latin1')
+
+	const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
+	if (metaCharset) { return metaCharset[1].toLowerCase() }
+
+	const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
+	if (httpEquiv) { return httpEquiv[1].toLowerCase() }
+
+	return null
+}
+
+const charsetFromBom = (buffer) => {
+	if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
+	if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
+	if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
+	return null
+}
+
+const decodeBody = (buffer, declaredCharset) => {
+	const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'
+
+	try {
+		return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
+	} catch (error) {
+		// Unknown/bogus label - utf-8 with replacement chars beats throwing.
+		return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
+	}
+}
+
+//
+// Read the body with a running byte count, aborting mid-stream once the cap is
+// passed. Content-Length is checked by the caller too, but it is optional and
+// can lie, so this is the cap that actually holds.
+//
+const readCapped = async (response, maxBytes, controller, url) => {
+	if (!response.body) { return Buffer.alloc(0) }
+
+	const chunks = []
+	let total = 0
+
+	for await (const chunk of response.body) {
+		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+		total += buffer.length
+
+		if (total > maxBytes) {
+			try { controller.abort() } catch (error) { /* already done */ }
+			throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
+		}
+
+		chunks.push(buffer)
+	}
+
+	return Buffer.concat(chunks, total)
+}
+
+//
+// One HTTP request, following redirects by hand.
+//
+// Manual redirects are the whole point: fetch's own 'follow' would hop to
+// 127.0.0.1 without ever re-consulting the safety gate.
+//
+const requestOnce = async (startUrl, options) => {
+	const {
+		maxBytes,
+		acceptMimes,
+		headersFor,
+		conditional,
+		timeoutMs,
+		deadline,
+		referer,
+	} = options
+
+	let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
+	const visited = new Set()
+
+	for (let hop = 0; hop <= config.maxRedirects; hop++) {
+		if (visited.has(currentUrl)) {
+			throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
+		}
+		visited.add(currentUrl)
+
+		// Re-validate on every hop. This is the check the old code never had.
+		try {
+			await UrlSafety.assertFetchable(currentUrl)
+		} catch (error) {
+			// A DNS failure is transient - the host may simply be down right
+			// now - so it must not be classified alongside "this address is
+			// private", which is a permanent refusal. Collapsing the two meant
+			// a briefly-unresolvable domain got parked for the full cache TTL
+			// and had its attachment row deleted.
+			const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
+			throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
+		}
+
+		const remaining = deadline - Date.now()
+		if (remaining <= 0) {
+			throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
+		}
+
+		const controller = new AbortController()
+		const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))
+
+		let response = null
+		try {
+			const headers = headersFor(currentUrl, referer)
+
+			if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
+			if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }
+
+			response = await fetch(currentUrl, {
+				method: 'GET',
+				redirect: 'manual',
+				signal: controller.signal,
+				headers,
+			})
+		} catch (error) {
+			clearTimeout(timer)
+
+			if (controller.signal.aborted) {
+				throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
+			}
+			throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
+		}
+
+		// Redirect?
+		if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
+			clearTimeout(timer)
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }
+
+			const location = response.headers.get('location')
+			let next = null
+			try {
+				next = new URL(location, currentUrl)
+			} catch (error) {
+				throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
+			}
+
+			// Never follow https -> http. A downgrade mid-chain is either an
+			// attack or a misconfiguration, and is not worth the preview.
+			if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
+				throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
+			}
+
+			currentUrl = next.href
+			continue
+		}
+
+		clearTimeout(timer)
+
+		if (response.status === 304) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			return { notModified: true, status: 304, finalUrl: currentUrl, response }
+		}
+
+		if (!response.ok) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
+				status: response.status,
+				retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
+				url: currentUrl,
+			})
+		}
+
+		const contentType = contentTypeOf(response)
+		const mime = mimeOf(contentType)
+
+		// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
+		// being handed to cheerio.
+		if (acceptMimes && !acceptMimes(mime)) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
+		}
+
+		const declaredLength = Number(response.headers.get('content-length'))
+		if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
+		}
+
+		const buffer = await readCapped(response, maxBytes, controller, currentUrl)
+
+		return {
+			notModified: false,
+			status: response.status,
+			finalUrl: currentUrl,
+			contentType,
+			mime,
+			buffer,
+			response,
+		}
+	}
+
+	throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
+}
+
+//
+// Retry wrapper. Only transient classes are retried - a 404 is an answer.
+//
+const withRetries = async (attemptFn, { retries, deadline }) => {
+	let lastError = null
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		try {
+			return await attemptFn()
+		} catch (error) {
+			lastError = error
+
+			if (!isRetryable(error) || attempt === retries) { throw error }
+
+			// A server asking us to come back in an hour is not something to
+			// sit on - surface it so the cache can park the URL instead.
+			if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }
+
+			const wait = error.retryAfterMs != null
+				? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
+				: backoffFor(attempt)
+
+			if (Date.now() + wait >= deadline) { throw error }
+
+			await sleep(wait)
+		}
+	}
+
+	throw lastError
+}
+
+//
+// Fetch an HTML document.
+//
+HttpFetch.fetchDocument = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.pageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxHtmlBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes: (mime) => HTML_TYPES.includes(mime),
+		headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
+			url: target,
+			// A browser arriving at a page has *some* referer; its own origin is
+			// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
+			referer: referer || originOf(target),
+		}),
+		conditional: options.conditional,
+		timeoutMs,
+		deadline,
+		referer: options.referer,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	if (result.notModified) {
+		return {
+			notModified: true,
+			status: 304,
+			finalUrl: result.finalUrl,
+			html: null,
+			profileId: profile.id,
+		}
+	}
+
+	const declaredCharset = charsetFromContentType(result.contentType)
+	const { text, charset } = decodeBody(result.buffer, declaredCharset)
+
+	return {
+		notModified: false,
+		status: result.status,
+		finalUrl: result.finalUrl,
+		html: text,
+		charset,
+		contentType: result.contentType,
+		bytes: result.buffer.length,
+		etag: result.response.headers.get('etag') || null,
+		lastModified: result.response.headers.get('last-modified') || null,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch a binary (thumbnail image).
+//
+HttpFetch.fetchBinary = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.imageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxImageBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const acceptMimes = (mime) => {
+		if (!mime.startsWith('image/')) { return false }
+		if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
+		return true
+	}
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes,
+		headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
+			url: target,
+			pageUrl: options.pageUrl,
+		}),
+		timeoutMs,
+		deadline,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	return {
+		status: result.status,
+		finalUrl: result.finalUrl,
+		contentType: result.contentType,
+		mime: result.mime,
+		buffer: result.buffer,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
+//
+HttpFetch.fetchText = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || 5000
+	const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await requestOnce(url, {
+		maxBytes: options.maxBytes || config.maxRobotsBytes,
+		acceptMimes: null, // robots.txt is served as every type under the sun
+		headersFor: () => BrowserProfiles.robotsHeaders(profile),
+		timeoutMs,
+		deadline,
+	})
+
+	if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }
+
+	const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
+	return { status: result.status, text, finalUrl: result.finalUrl }
+}
+
+const hostnameOf = (url) => {
+	try { return new URL(typeof url === 'string' ? url : url.href).hostname }
+	catch (error) { return '' }
+}
+
+const originOf = (url) => {
+	try { return new URL(url).origin + '/' }
+	catch (error) { return undefined }
+}
+
+//
+// Deliberately NOT attempted here, so nobody wonders later:
+//   - Header ordering: fetch normalises it; controlling order needs raw sockets.
+//   - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
+//   - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
+//     rather than evasive.
+//

server/helpers/Realtime.js

@@ -0,0 +1,43 @@
+//
+// Socket.IO emit wrapper.
+//
+// Replaces the `if(typeof SocketIo != 'undefined')` guards that were scattered
+// through the models. Those guards did not just avoid a crash - they silently
+// disabled ALL scraping under Jest, which is why none of this code was ever
+// testable. Here the no-socket case is a no-op rather than a skipped branch, so
+// the surrounding logic runs identically in tests and in production.
+//
+
+let Realtime = module.exports = {}
+
+let warned = false
+
+const server = () => (typeof global.SocketIo !== 'undefined' && global.SocketIo) ? global.SocketIo : null
+
+Realtime.isEnabled = () => server() !== null
+
+const emit = (room, event, payload) => {
+	const io = server()
+
+	if (!io) {
+		if (!warned) {
+			warned = true
+			console.log('[realtime] no socket server bound; emits are no-ops')
+		}
+		return false
+	}
+
+	try {
+		if (payload === undefined) { io.to(room).emit(event) }
+		else { io.to(room).emit(event, payload) }
+		return true
+	} catch (error) {
+		console.log('[realtime] emit failed', event, error.message)
+		return false
+	}
+}
+
+Realtime.emitToUser = (userId, event, payload) => emit(`user:${userId}`, event, payload)
+Realtime.emitToNote = (noteRawTextId, event, payload) => emit(`note:${noteRawTextId}`, event, payload)
+
+Realtime.__resetWarning = () => { warned = false }

server/helpers/Robots.js

@@ -0,0 +1,216 @@
+//
+// robots.txt fetching, parsing and caching.
+//
+// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
+// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
+// a 404 or a flaky server is not a refusal and these are single-page fetches of
+// URLs a user explicitly saved.
+//
+
+const config = require('@config/scrape')
+const HttpFetch = require('@helpers/HttpFetch')
+
+let Robots = module.exports = {}
+
+// origin -> { rules, expiresAt }
+const cache = new Map()
+
+Robots.__resetCache = () => cache.clear()
+Robots.__cacheSize = () => cache.size
+
+const ALLOW_ALL = { groups: [], allowAll: true }
+
+//
+// Parse robots.txt into user-agent groups.
+//
+// Consecutive User-agent lines share one rule block, which is the part naive
+// parsers get wrong.
+//
+Robots.parse = (text) => {
+	if (!text || typeof text !== 'string') { return ALLOW_ALL }
+
+	const groups = []
+	let current = null
+	let expectingAgents = false
+
+	for (const rawLine of text.split(/\r?\n/)) {
+		const line = rawLine.replace(/#.*$/, '').trim()
+		if (!line) { continue }
+
+		const separator = line.indexOf(':')
+		if (separator === -1) { continue }
+
+		const field = line.slice(0, separator).trim().toLowerCase()
+		const value = line.slice(separator + 1).trim()
+
+		if (field === 'user-agent') {
+			// A User-agent line directly after another starts a shared group.
+			if (!current || !expectingAgents) {
+				current = { agents: [], rules: [], crawlDelayMs: null }
+				groups.push(current)
+			}
+			current.agents.push(value.toLowerCase())
+			expectingAgents = true
+			continue
+		}
+
+		if (!current) { continue } // directive before any user-agent
+		expectingAgents = false
+
+		if (field === 'disallow' || field === 'allow') {
+			// "Disallow:" with an empty value means allow everything.
+			if (field === 'disallow' && value === '') { continue }
+			current.rules.push({ allow: field === 'allow', path: value })
+			continue
+		}
+
+		if (field === 'crawl-delay') {
+			const seconds = Number(value)
+			if (Number.isFinite(seconds) && seconds >= 0) {
+				current.crawlDelayMs = Math.round(seconds * 1000)
+			}
+		}
+	}
+
+	return { groups, allowAll: groups.length === 0 }
+}
+
+//
+// Pick the group whose agent token best matches us. A named match always beats
+// the "*" wildcard, regardless of file order.
+//
+const groupFor = (rules, userAgentToken) => {
+	if (!rules || rules.allowAll) { return null }
+
+	const token = userAgentToken.toLowerCase()
+	let best = null
+	let bestScore = -1
+
+	for (const group of rules.groups) {
+		for (const agent of group.agents) {
+			let score = -1
+			if (agent === '*') { score = 0 }
+			else if (token.includes(agent)) { score = agent.length }
+
+			if (score > bestScore) {
+				bestScore = score
+				best = group
+			}
+		}
+	}
+
+	return best
+}
+
+//
+// Match a robots path pattern, supporting the * and $ wildcards.
+//
+const matchesPattern = (pattern, path) => {
+	if (pattern === '') { return false }
+	if (!pattern.includes('*') && !pattern.includes('$')) {
+		return path.startsWith(pattern)
+	}
+
+	const anchored = pattern.endsWith('$')
+	const body = anchored ? pattern.slice(0, -1) : pattern
+
+	const expression = body
+		.split('*')
+		.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+		.join('.*')
+
+	return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
+}
+
+//
+// Longest matching rule wins; Allow beats Disallow at equal length.
+//
+Robots.isPathAllowed = (rules, path, userAgentToken) => {
+	const group = groupFor(rules, userAgentToken)
+	if (!group) { return true }
+
+	let decision = true
+	let decisionLength = -1
+
+	for (const rule of group.rules) {
+		if (!matchesPattern(rule.path, path)) { continue }
+
+		const length = rule.path.length
+		if (length > decisionLength || (length === decisionLength && rule.allow)) {
+			decision = rule.allow
+			decisionLength = length
+		}
+	}
+
+	return decision
+}
+
+const evict = () => {
+	if (cache.size <= config.robotsMaxOrigins) { return }
+	// Map preserves insertion order, so the oldest key is first.
+	const oldest = cache.keys().next().value
+	cache.delete(oldest)
+}
+
+const getRules = async (origin) => {
+	const cached = cache.get(origin)
+	if (cached && cached.expiresAt > Date.now()) { return cached.rules }
+
+	let rules = ALLOW_ALL
+	let ttl = config.robotsTtlMs
+
+	try {
+		const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
+			maxBytes: config.maxRobotsBytes,
+		})
+		rules = Robots.parse(result.text)
+	} catch (error) {
+		// No robots.txt, or we could not reach it. Permissive, but cached for a
+		// short while only so a transient outage does not grant a day's pass.
+		rules = ALLOW_ALL
+		ttl = (error && error.kind === 'http' && error.status === 404)
+			? config.robotsNegativeTtlMs
+			: config.robotsErrorTtlMs
+	}
+
+	cache.set(origin, { rules, expiresAt: Date.now() + ttl })
+	evict()
+
+	return rules
+}
+Robots.__getRules = getRules
+
+const originAndPath = (url) => {
+	const parsed = new URL(typeof url === 'string' ? url : url.href)
+	return { origin: parsed.origin, path: parsed.pathname + parsed.search }
+}
+
+//
+// Is this URL allowed by its host's robots.txt?
+//
+Robots.isAllowed = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return false }
+
+	const rules = await getRules(parts.origin)
+	return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
+}
+
+//
+// Crawl-delay for this host, or null. Capped: a site asking for an hour between
+// requests gets its job failed rather than parked in the queue forever.
+//
+Robots.crawlDelayMs = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return null }
+
+	const rules = await getRules(parts.origin)
+	const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())
+
+	if (!group || group.crawlDelayMs == null) { return null }
+	return group.crawlDelayMs
+}
+
+Robots.exceedsCrawlDelayCap = (delayMs) => {
+	return delayMs != null && delayMs > config.maxCrawlDelayMs
+}

server/helpers/Schema.js

@@ -0,0 +1,104 @@
+//
+// Idempotent schema convergence for the scraper tables.
+//
+// There is no migration framework in this project and no schema file in git,
+// so this runs at boot, checks information_schema for each object, and applies
+// only what is missing. Dev and prod self-heal, and deploy.sh needs no change.
+//
+// Portability matters here: dev is MariaDB 11.4 and prod is MySQL 8.
+// ADD COLUMN IF NOT EXISTS is MariaDB-only and is deliberately not used -
+// every ALTER is guarded by an explicit existence check instead.
+//
+
+const db = require('@config/database')
+
+let Schema = module.exports = {}
+
+const SCRAPE_CACHE_TABLE = `
+	CREATE TABLE IF NOT EXISTS scrape_cache (
+		id            int(10) unsigned NOT NULL AUTO_INCREMENT,
+		url_hash      char(64) NOT NULL,
+		url           varchar(1000) NOT NULL DEFAULT '',
+		canonical_url varchar(1000) DEFAULT NULL,
+		status        smallint(6) NOT NULL DEFAULT 0,
+		title         varchar(512) DEFAULT NULL,
+		site_name     varchar(255) DEFAULT NULL,
+		description   mediumtext DEFAULT NULL,
+		keywords      varchar(512) DEFAULT NULL,
+		search_text   mediumtext DEFAULT NULL,
+		image_url     varchar(1000) DEFAULT NULL,
+		file_location varchar(500) DEFAULT NULL,
+		etag          varchar(255) DEFAULT NULL,
+		last_modified varchar(128) DEFAULT NULL,
+		content_type  varchar(128) DEFAULT NULL,
+		fetch_state   varchar(20) NOT NULL DEFAULT 'pending',
+		failure_count int(11) NOT NULL DEFAULT 0,
+		last_error    varchar(255) DEFAULT NULL,
+		fetched_at    int(11) DEFAULT NULL,
+		expires_at    int(11) DEFAULT NULL,
+		PRIMARY KEY (id),
+		UNIQUE KEY scrape_cache_url_hash (url_hash),
+		KEY scrape_cache_expires_at (expires_at),
+		KEY scrape_cache_fetch_state (fetch_state)
+	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+`
+
+// DEFAULT 2 ("done") keeps existing rows out of the boot recovery sweep.
+const ATTACHMENT_COLUMNS = [
+	{ name: 'scrape_state', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_state tinyint(4) NOT NULL DEFAULT 2' },
+	{ name: 'scrape_attempts', ddl: 'ALTER TABLE attachment ADD COLUMN scrape_attempts tinyint(4) NOT NULL DEFAULT 0' },
+	{ name: 'enqueued_at', ddl: 'ALTER TABLE attachment ADD COLUMN enqueued_at int(11) DEFAULT NULL' },
+]
+
+const ATTACHMENT_INDEXES = [
+	{ name: 'attachment_scrape_state', ddl: 'ALTER TABLE attachment ADD KEY attachment_scrape_state (scrape_state, enqueued_at)' },
+]
+
+const columnExists = async (table, column) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.COLUMNS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ? LIMIT 1`,
+		[table, column]
+	)
+	return rows.length > 0
+}
+
+const indexExists = async (table, index) => {
+	const [rows] = await db.promise().query(
+		`SELECT 1 FROM information_schema.STATISTICS
+		 WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1`,
+		[table, index]
+	)
+	return rows.length > 0
+}
+Schema.__columnExists = columnExists
+Schema.__indexExists = indexExists
+
+//
+// Safe to call repeatedly. Returns the list of statements it actually applied,
+// so boot logging says something useful on the one run that matters.
+//
+Schema.ensureScrapeSchema = async () => {
+	const applied = []
+
+	await db.promise().query(SCRAPE_CACHE_TABLE)
+
+	for (const column of ATTACHMENT_COLUMNS) {
+		if (await columnExists('attachment', column.name)) { continue }
+		await db.promise().query(column.ddl)
+		applied.push(`attachment.${column.name}`)
+	}
+
+	// Index last: it references the columns above.
+	for (const index of ATTACHMENT_INDEXES) {
+		if (await indexExists('attachment', index.name)) { continue }
+		await db.promise().query(index.ddl)
+		applied.push(`attachment KEY ${index.name}`)
+	}
+
+	if (applied.length) {
+		console.log('[schema] applied scrape schema changes:', applied.join(', '))
+	}
+
+	return applied
+}

server/helpers/UrlSafety.js

@@ -0,0 +1,246 @@
+//
+// URL normalisation and the SSRF gate.
+//
+// Scraped URLs come from untrusted note text and from the /pushmebaby query
+// param, so every URL is validated before a socket is opened - and again after
+// every redirect hop, which is the hop that actually matters (a public URL that
+// 302s to 127.0.0.1 is the classic bypass).
+//
+// Pure except for assertFetchable, which resolves DNS.
+//
+
+const crypto = require('crypto')
+const dns = require('dns').promises
+const net = require('net')
+const config = require('@config/scrape')
+
+let UrlSafety = module.exports = {}
+
+const FETCHABLE_SCHEMES = new Set(['http:', 'https:'])
+
+// Hostname suffixes that never belong to a public site.
+const BLOCKED_SUFFIXES = ['.localhost', '.local', '.internal', '.home.arpa', '.lan']
+
+// Query params that change nothing about the page but would fragment the cache.
+const TRACKING_PARAMS = [
+	/^utm_/i, /^fbclid$/i, /^gclid$/i, /^dclid$/i, /^msclkid$/i,
+	/^mc_(e|c)id$/i, /^igshid$/i, /^ref_src$/i, /^_hs(enc|mi)$/i, /^vero_/i,
+]
+
+class SafetyError extends Error {
+	constructor(reason, message) {
+		super(message || reason)
+		this.name = 'SafetyError'
+		this.reason = reason
+	}
+}
+UrlSafety.SafetyError = SafetyError
+
+//
+// Parse to a URL object, or null. Does NOT mutate anything that would change
+// which resource is addressed - callers may store the result.
+//
+UrlSafety.parse = (raw, base) => {
+	if (!raw || typeof raw !== 'string') { return null }
+
+	let url = null
+	try {
+		url = new URL(raw.trim(), base || undefined)
+	} catch (error) {
+		return null
+	}
+
+	if (!FETCHABLE_SCHEMES.has(url.protocol)) { return null }
+	if (url.href.length > config.maxUrlLength) { return null }
+
+	return url
+}
+
+//
+// Canonical form used ONLY for cache keys. Never store this - it deliberately
+// discards tracking params and case, which would change what the user wrote.
+//
+UrlSafety.normalizeForKey = (raw) => {
+	const url = UrlSafety.parse(raw)
+	if (!url) { return null }
+
+	url.hash = ''
+	url.username = ''
+	url.password = ''
+
+	// Trailing dot on the host is legal DNS but splits the cache key.
+	if (url.hostname.endsWith('.')) {
+		url.hostname = url.hostname.slice(0, -1)
+	}
+	url.hostname = url.hostname.toLowerCase()
+
+	// new URL already strips the default port, but be explicit.
+	if ((url.protocol === 'http:' && url.port === '80') ||
+		(url.protocol === 'https:' && url.port === '443')) {
+		url.port = ''
+	}
+
+	for (const key of [...url.searchParams.keys()]) {
+		if (TRACKING_PARAMS.some(pattern => pattern.test(key))) {
+			url.searchParams.delete(key)
+		}
+	}
+	url.searchParams.sort()
+
+	if (url.pathname === '') { url.pathname = '/' }
+
+	return url
+}
+
+// sha256 of the normalised href. This is scrape_cache.url_hash - char(64),
+// because a UNIQUE index on varchar(1000) utf8mb4 would be 4000 bytes and
+// blow InnoDB's 3072-byte key limit.
+UrlSafety.urlKey = (raw) => {
+	const url = UrlSafety.normalizeForKey(raw)
+	if (!url) { return null }
+	return crypto.createHash('sha256').update(url.href).digest('hex')
+}
+
+//
+// True when two URLs address the same resource modulo scheme, "www." and a
+// trailing slash.
+//
+// Load-bearing: scanTextForWebsites diffs the URLs currently in a note against
+// the URLs already stored as attachments. Raw string equality would treat
+// http://foo.com and https://foo.com as different resources and delete-then-
+// rescrape the row on every save.
+//
+UrlSafety.sameResource = (a, b) => {
+	const left = UrlSafety.normalizeForKey(a)
+	const right = UrlSafety.normalizeForKey(b)
+	if (!left || !right) { return false }
+
+	const host = (url) => url.hostname.replace(/^www\./, '')
+	const path = (url) => url.pathname.replace(/\/+$/, '')
+
+	return host(left) === host(right) &&
+		path(left) === path(right) &&
+		left.search === right.search
+}
+
+UrlSafety.registrableHost = (hostname) => {
+	return String(hostname || '').toLowerCase().replace(/^www\./, '').replace(/\.$/, '')
+}
+
+//
+// Private / non-routable address detection.
+//
+// Covers the ranges that matter for SSRF against this app: loopback, RFC1918,
+// link-local (including the 169.254.169.254 cloud metadata endpoint), CGNAT,
+// and the IPv6 equivalents.
+//
+UrlSafety.isPrivateIp = (address) => {
+	if (!address) { return true }
+
+	const version = net.isIP(address)
+	if (version === 0) { return true } // not an IP - caller resolves first
+
+	if (version === 4) { return isPrivateIpv4(address) }
+	return isPrivateIpv6(address)
+}
+
+const isPrivateIpv4 = (address) => {
+	const parts = address.split('.').map(Number)
+	if (parts.length !== 4 || parts.some(n => !Number.isInteger(n) || n < 0 || n > 255)) {
+		return true
+	}
+	const [a, b] = parts
+
+	if (a === 0) { return true }                            // 0.0.0.0/8 "this host"
+	if (a === 10) { return true }                           // RFC1918
+	if (a === 127) { return true }                          // loopback
+	if (a === 169 && b === 254) { return true }             // link-local + metadata
+	if (a === 172 && b >= 16 && b <= 31) { return true }    // RFC1918
+	if (a === 192 && b === 168) { return true }             // RFC1918
+	if (a === 192 && b === 0) { return true }               // 192.0.0/24 + 192.0.2/24
+	if (a === 198 && (b === 18 || b === 19)) { return true } // benchmarking
+	if (a === 100 && b >= 64 && b <= 127) { return true }   // CGNAT
+	if (a === 198 && b === 51) { return true }              // TEST-NET-2
+	if (a === 203 && b === 0) { return true }               // TEST-NET-3
+	if (a >= 224) { return true }                           // multicast + reserved
+
+	return false
+}
+
+const isPrivateIpv6 = (address) => {
+	const lower = address.toLowerCase().replace(/^\[|\]$/g, '')
+
+	// Unwrap v4-mapped (::ffff:127.0.0.1) and NAT64 (64:ff9b::/96) before
+	// judging, otherwise ::ffff:127.0.0.1 reads as "some v6 address".
+	const mapped = lower.match(/^(?:::ffff:|64:ff9b::)(\d+\.\d+\.\d+\.\d+)$/)
+	if (mapped) { return isPrivateIpv4(mapped[1]) }
+
+	if (lower === '::' || lower === '::1') { return true }   // unspecified, loopback
+	if (/^f[cd]/.test(lower)) { return true }               // fc00::/7 unique-local
+	if (/^fe[89ab]/.test(lower)) { return true }            // fe80::/10 link-local
+	if (/^ff/.test(lower)) { return true }                  // multicast
+
+	return false
+}
+
+//
+// The gate. Throws SafetyError; resolves to the vetted addresses on success.
+//
+// Known limitation (documented, accepted): this resolves DNS and then hands the
+// hostname to fetch, which resolves again. A hostile 0-TTL record can answer
+// differently the second time. Closing that properly needs a custom undici
+// Agent with connect.lookup - deliberately deferred, see the plan's Phase 5.
+// Residual risk is blind SSRF only: no credentials are sent and no response
+// body is returned to whoever planted the URL.
+//
+UrlSafety.assertFetchable = async (raw) => {
+	const url = raw instanceof URL ? raw : UrlSafety.parse(raw)
+
+	if (!url) {
+		throw new SafetyError('bad_url', `Not a fetchable URL: ${raw}`)
+	}
+	if (!FETCHABLE_SCHEMES.has(url.protocol)) {
+		throw new SafetyError('bad_scheme', `Refusing scheme ${url.protocol}`)
+	}
+	if (url.username || url.password) {
+		throw new SafetyError('credentials_in_url', 'Refusing URL with embedded credentials')
+	}
+
+	const hostname = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase()
+
+	if (!hostname) {
+		throw new SafetyError('bad_url', 'URL has no host')
+	}
+	if (hostname === 'localhost' || BLOCKED_SUFFIXES.some(suffix => hostname.endsWith(suffix))) {
+		throw new SafetyError('blocked_host', `Refusing internal host ${hostname}`)
+	}
+
+	// Literal IP in the URL - no DNS needed.
+	if (net.isIP(hostname)) {
+		if (UrlSafety.isPrivateIp(hostname)) {
+			throw new SafetyError('private_ip', `Refusing non-routable address ${hostname}`)
+		}
+		return [hostname]
+	}
+
+	let resolved = []
+	try {
+		resolved = await dns.lookup(hostname, { all: true, verbatim: true })
+	} catch (error) {
+		throw new SafetyError('dns_failed', `Could not resolve ${hostname}: ${error.code || error.message}`)
+	}
+
+	if (!resolved.length) {
+		throw new SafetyError('dns_failed', `No addresses for ${hostname}`)
+	}
+
+	// Every answer must be public. One private address is enough to refuse -
+	// otherwise a round-robin record gets us some of the time.
+	for (const entry of resolved) {
+		if (UrlSafety.isPrivateIp(entry.address)) {
+			throw new SafetyError('private_ip', `${hostname} resolves to non-routable ${entry.address}`)
+		}
+	}
+
+	return resolved.map(entry => entry.address)
+}

server/migrations/0001_scrape_cache_and_state.sql

@@ -0,0 +1,51 @@
+--
+-- Scrape cache + per-attachment scrape lifecycle.
+--
+-- This file is the human-readable record. It is NOT applied by hand: the same
+-- statements live in server/helpers/Schema.js and are applied idempotently at
+-- boot after an information_schema existence check, so dev and prod converge
+-- without a migration runner.
+--
+-- Portability: dev is MariaDB 11.4, prod is MySQL 8. Nothing here may use
+-- ADD COLUMN IF NOT EXISTS (MariaDB-only).
+--
+
+-- One row per URL, shared across users and notes. url_hash carries the UNIQUE
+-- index because a UNIQUE on varchar(1000) utf8mb4 is 4000 bytes and exceeds
+-- InnoDB's 3072-byte key limit.
+CREATE TABLE IF NOT EXISTS `scrape_cache` (
+  `id`            int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `url_hash`      char(64) NOT NULL,
+  `url`           varchar(1000) NOT NULL DEFAULT '',
+  `canonical_url` varchar(1000) DEFAULT NULL,
+  `status`        smallint(6) NOT NULL DEFAULT 0,
+  `title`         varchar(512) DEFAULT NULL,
+  `site_name`     varchar(255) DEFAULT NULL,
+  `description`   mediumtext DEFAULT NULL,
+  `keywords`      varchar(512) DEFAULT NULL,
+  `search_text`   mediumtext DEFAULT NULL,
+  `image_url`     varchar(1000) DEFAULT NULL,
+  `file_location` varchar(500) DEFAULT NULL,
+  `etag`          varchar(255) DEFAULT NULL,
+  `last_modified` varchar(128) DEFAULT NULL,
+  `content_type`  varchar(128) DEFAULT NULL,
+  `fetch_state`   varchar(20) NOT NULL DEFAULT 'pending',
+  `failure_count` int(11) NOT NULL DEFAULT 0,
+  `last_error`    varchar(255) DEFAULT NULL,
+  `fetched_at`    int(11) DEFAULT NULL,
+  `expires_at`    int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `scrape_cache_url_hash` (`url_hash`),
+  KEY `scrape_cache_expires_at` (`expires_at`),
+  KEY `scrape_cache_fetch_state` (`fetch_state`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Per-attachment lifecycle, so an in-memory queue survives a restart.
+--
+-- scrape_state: 0 = pending, 1 = fetching, 2 = done, 3 = failed.
+-- DEFAULT 2 is load-bearing: it keeps the ~2200 pre-existing rows out of the
+-- recovery sweep on first boot.
+ALTER TABLE `attachment` ADD COLUMN `scrape_state`    tinyint(4) NOT NULL DEFAULT 2;
+ALTER TABLE `attachment` ADD COLUMN `scrape_attempts` tinyint(4) NOT NULL DEFAULT 0;
+ALTER TABLE `attachment` ADD COLUMN `enqueued_at`     int(11) DEFAULT NULL;
+ALTER TABLE `attachment` ADD KEY `attachment_scrape_state` (`scrape_state`, `enqueued_at`);

server/models/ScrapeCache.js

@@ -0,0 +1,224 @@
+//
+// Read-through cache for scraped URL metadata, keyed by a hash of the
+// normalised URL and shared across users and notes.
+//
+// A fresh hit means zero network I/O. A stale hit means a conditional request
+// (If-None-Match / If-Modified-Since), so an unchanged page costs one 304
+// rather than a full page load. Failures are backed off so a dead link is not
+// refetched on every note save.
+//
+// Stores only public page metadata. Note content never reaches this table.
+//
+
+const db = require('@config/database')
+const UrlSafety = require('@helpers/UrlSafety')
+
+let ScrapeCache = module.exports = {}
+
+const SUCCESS_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
+
+// Widening backoff on repeated failure, then give up and stay failed.
+const FAILURE_BACKOFF_SECONDS = [60 * 60, 6 * 60 * 60, 24 * 60 * 60]
+const MAX_FAILURES = FAILURE_BACKOFF_SECONDS.length
+
+ScrapeCache.SUCCESS_TTL_SECONDS = SUCCESS_TTL_SECONDS
+ScrapeCache.MAX_FAILURES = MAX_FAILURES
+
+const now = () => Math.round(Date.now() / 1000)
+
+const truncate = (value, length) => {
+	if (value == null) { return null }
+	const text = String(value)
+	return text.length > length ? text.substring(0, length) : text
+}
+
+ScrapeCache.get = async (urlOrKey) => {
+	const key = /^[0-9a-f]{64}$/.test(urlOrKey) ? urlOrKey : UrlSafety.urlKey(urlOrKey)
+	if (!key) { return null }
+
+	const [rows] = await db.promise().query('SELECT * FROM scrape_cache WHERE url_hash = ? LIMIT 1', [key])
+	return rows.length ? rows[0] : null
+}
+
+// Fresh = inside its TTL and successfully fetched.
+ScrapeCache.isFresh = (row, at) => {
+	if (!row || !row.expires_at) { return false }
+	return row.expires_at > (at || now())
+}
+
+ScrapeCache.isUsable = (row) => {
+	return !!(row && row.fetch_state === 'ok' && row.search_text != null)
+}
+
+// Has this URL failed enough times that we should stop trying?
+ScrapeCache.isExhausted = (row) => {
+	return !!(row && row.fetch_state === 'failed' && row.failure_count >= MAX_FAILURES)
+}
+
+//
+// Conditional-request headers for a stale-but-successful row. Null when there
+// is nothing to revalidate against.
+//
+ScrapeCache.conditionalFor = (row) => {
+	if (!row || row.fetch_state !== 'ok') { return null }
+	if (!row.etag && !row.last_modified) { return null }
+
+	return { etag: row.etag || null, lastModified: row.last_modified || null }
+}
+
+//
+// Reserve a row before fetching, so a concurrent save for the same URL finds
+// it rather than starting a second fetch.
+//
+ScrapeCache.markPending = async (url) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache (url_hash, url, fetch_state, fetched_at)
+		 VALUES (?, ?, 'pending', ?)
+		 ON DUPLICATE KEY UPDATE fetch_state = IF(fetch_state = 'ok', fetch_state, 'pending')`,
+		[key, truncate(url, 1000), now()]
+	)
+
+	return key
+}
+
+//
+// Store a successful scrape.
+//
+ScrapeCache.putSuccess = async (url, result) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const at = now()
+	const fields = {
+		url: truncate(url, 1000),
+		canonical_url: truncate(result.canonicalUrl, 1000),
+		status: result.status || 200,
+		title: truncate(result.title, 512),
+		site_name: truncate(result.siteName, 255),
+		description: result.description || null,
+		keywords: truncate(result.keywords, 512),
+		search_text: result.searchText || null,
+		image_url: truncate(result.imageUrl, 1000),
+		file_location: truncate(result.fileLocation, 500),
+		etag: truncate(result.etag, 255),
+		last_modified: truncate(result.lastModified, 128),
+		content_type: truncate(result.contentType, 128),
+		fetch_state: 'ok',
+		failure_count: 0,
+		last_error: null,
+		fetched_at: at,
+		expires_at: at + SUCCESS_TTL_SECONDS,
+	}
+
+	const columns = Object.keys(fields)
+	const values = columns.map(column => fields[column])
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache (url_hash, ${columns.join(', ')})
+		 VALUES (?, ${columns.map(() => '?').join(', ')})
+		 ON DUPLICATE KEY UPDATE ${columns.map(column => `${column} = VALUES(${column})`).join(', ')}`,
+		[key, ...values]
+	)
+
+	return key
+}
+
+//
+// A 304. The page is unchanged, so only the freshness window moves.
+//
+// Deliberately does NOT touch attachment.last_indexed - Attachment.search
+// orders by it, so revalidating would silently reshuffle the user's list.
+//
+ScrapeCache.touchNotModified = async (url) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const at = now()
+	await db.promise().query(
+		`UPDATE scrape_cache
+		 SET fetched_at = ?, expires_at = ?, fetch_state = 'ok', failure_count = 0, last_error = NULL
+		 WHERE url_hash = ?`,
+		[at, at + SUCCESS_TTL_SECONDS, key]
+	)
+
+	return key
+}
+
+//
+// Record a failure with a widening retry window.
+//
+// `blocked` (robots or the safety gate) is terminal: retrying will not change
+// the answer, so it is parked for the full success TTL instead of backing off.
+//
+ScrapeCache.recordFailure = async (url, error) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return null }
+
+	const existing = await ScrapeCache.get(key)
+	const failureCount = (existing ? existing.failure_count : 0) + 1
+
+	const kind = (error && error.kind) || 'unknown'
+	const terminal = kind === 'blocked' || kind === 'robots_disallowed'
+
+	let ttl = null
+	if (terminal) {
+		ttl = SUCCESS_TTL_SECONDS
+	} else if (error && error.retryAfterMs) {
+		ttl = Math.round(error.retryAfterMs / 1000)
+	} else {
+		ttl = FAILURE_BACKOFF_SECONDS[Math.min(failureCount, MAX_FAILURES) - 1]
+	}
+
+	const at = now()
+	const state = terminal ? 'blocked' : 'failed'
+
+	await db.promise().query(
+		`INSERT INTO scrape_cache
+			(url_hash, url, status, fetch_state, failure_count, last_error, fetched_at, expires_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		 ON DUPLICATE KEY UPDATE
+			status = VALUES(status),
+			fetch_state = VALUES(fetch_state),
+			failure_count = VALUES(failure_count),
+			last_error = VALUES(last_error),
+			fetched_at = VALUES(fetched_at),
+			expires_at = VALUES(expires_at)`,
+		[
+			key,
+			truncate(url, 1000),
+			(error && error.status) || 0,
+			state,
+			failureCount,
+			truncate(`${kind}: ${(error && error.message) || ''}`, 255),
+			at,
+			at + ttl,
+		]
+	)
+
+	return key
+}
+
+//
+// Project a cache row onto the shape Attachment rows want.
+//
+ScrapeCache.toAttachmentFields = (row) => {
+	if (!row) { return null }
+	return {
+		text: row.search_text || '',
+		fileLocation: row.file_location || null,
+		url: row.canonical_url || row.url,
+	}
+}
+
+ScrapeCache.purgeExpired = async (limit = 500) => {
+	const [result] = await db.promise().query(
+		`DELETE FROM scrape_cache
+		 WHERE expires_at IS NOT NULL AND expires_at < ? AND fetch_state <> 'ok'
+		 LIMIT ?`,
+		[now(), limit]
+	)
+	return result.affectedRows || 0
+}

server/tests/fixtures/html/README.md

@@ -0,0 +1,7 @@
+# HTML fixtures
+
+Frozen, hand-trimmed snapshots used by `server/tests/helpers/SiteScrape.test.js`.
+
+These are **not** fetched live. Each file isolates one parsing situation so a
+failure points at a specific branch. Add a new file rather than growing an
+existing one when covering a new case.

server/tests/fixtures/html/apple-touch-icon.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head>
+  <title>Icons By Size</title>
+  <link rel="icon" href="https://example.com/small.png" sizes="16x16">
+  <link rel="apple-touch-icon" href="https://example.com/touch-180.png" sizes="180x180">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/extensionless-og-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Generated OG Image</title>
+  <meta property="og:image" content="https://example.com/api/og?title=Hello&amp;theme=dark">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/h1-fallback.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<html>
+<head></head>
+<body><h1>Heading Is All We Have</h1><p>No title element, no meta tags.</p></body>
+</html>

server/tests/fixtures/html/imgs-missing-src.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html>
+<head><title>Broken Images</title></head>
+<body>
+  <img>
+  <img alt="no src here">
+  <img src="">
+  <img data-src="https://cdn.example.com/lazy.jpg">
+</body>
+</html>

server/tests/fixtures/html/jsonld-article.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+<head>
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@type": "NewsArticle",
+    "headline": "JSON-LD Headline",
+    "description": "Described in structured data.",
+    "image": ["https://cdn.example.com/ld-primary.jpg", "https://cdn.example.com/ld-second.jpg"]
+  }
+  </script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/jsonld-graph.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+<head>
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@graph": [
+      { "@type": "Organization", "name": "Not The Article" },
+      { "@type": "BlogPosting", "headline": "Graph Headline", "image": { "url": "https://cdn.example.com/graph.png" } }
+    ]
+  }
+  </script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/jsonld-malformed.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+<head>
+  <title>Survived Bad JSON</title>
+  <script type="application/ld+json">
+  { "@type": "Article", "headline": "unterminated,,, }
+  </script>
+  <script type="application/ld+json"></script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/keyword-content.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+<head><title>Beekeeping Guide</title></head>
+<body>
+  <nav>Home About Contact Login Signup Navigation Navigation</nav>
+  <script>var tracking = "tracking tracking tracking tracking tracking";</script>
+  <style>.beehive { color: red }</style>
+  <article class="content">
+    <p>Beekeeping is rewarding. Beekeeping requires patience. A beehive needs space.</p>
+    <p>The beehive must stay dry. Honey harvest comes later. Honey harvest rewards patience.</p>
+    <p>Beekeeping beginners should buy a beehive and study honey production carefully.</p>
+    <p>Honey production depends on the beehive and on steady beekeeping habits.</p>
+  </article>
+  <footer>Footer footer footer links copyright</footer>
+</body>
+</html>

server/tests/fixtures/html/malformed-unclosed.html

@@ -0,0 +1,7 @@
+<!doctype html>
+<html>
+<head><title>Unclosed Tags
+<body>
+  <div><p>Paragraph never closed
+  <ul><li>item one<li>item two
+</html>

server/tests/fixtures/html/no-metadata.html

@@ -0,0 +1,2 @@
+<!doctype html>
+<html><head></head><body></body></html>

server/tests/fixtures/html/og-complete.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <title>Fallback Title Nobody Should See</title>
+  <meta property="og:title" content="The Real Title">
+  <meta property="og:description" content="A proper social description of the page.">
+  <meta property="og:site_name" content="Example Journal">
+  <meta property="og:image" content="https://cdn.example.com/hero.jpg">
+  <meta property="og:url" content="https://example.com/canonical-post">
+  <meta name="description" content="A lesser meta description.">
+  <link rel="canonical" href="https://example.com/canonical-post">
+</head>
+<body><article><p>Body text.</p></article></body>
+</html>

server/tests/fixtures/html/protocol-relative-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Protocol Relative</title>
+  <meta property="og:image" content="//cdn.example.com/proto.jpg">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/relative-images.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head>
+  <title>Relative Assets</title>
+  <link rel="canonical" href="https://example.com/blog/2026/post">
+  <meta property="og:image" content="../images/hero.png">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/root-relative-icon.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Icon Only</title>
+  <link rel="shortcut icon" href="/favicon.ico">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/sized-images.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head><title>Sized</title></head>
+<body>
+  <img src="https://cdn.example.com/thumb.jpg" width="80" height="60">
+  <img src="https://cdn.example.com/big.jpg" width="1200" height="800">
+  <img src="https://cdn.example.com/mid.jpg" width="400" height="300">
+</body>
+</html>

server/tests/fixtures/html/srcset.html

@@ -0,0 +1,7 @@
+<!doctype html>
+<html>
+<head><title>Srcset</title></head>
+<body>
+  <img srcset="https://cdn.example.com/small.jpg 320w, https://cdn.example.com/large.jpg 1600w, https://cdn.example.com/medium.jpg 800w">
+</body>
+</html>

server/tests/fixtures/html/svg-only-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>SVG Only</title>
+  <meta property="og:image" content="https://cdn.example.com/logo.svg">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/title-only.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<html>
+<head><title>   Just   A   Title   </title></head>
+<body><h1>An H1 Heading</h1><p>No metadata here at all.</p></body>
+</html>

server/tests/fixtures/html/tracking-pixels.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head><title>Pixels</title></head>
+<body>
+  <img src="https://analytics.example.net/1x1.gif" width="1" height="1">
+  <img src="https://cdn.example.com/spacer.gif">
+  <img src="https://cdn.example.com/real-photo.jpg" width="800" height="600">
+</body>
+</html>

server/tests/fixtures/html/twitter-only.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html>
+<head>
+  <title>Document Title</title>
+  <meta name="twitter:title" content="Twitter Card Title">
+  <meta name="twitter:description" content="Twitter card description.">
+  <meta name="twitter:image" content="https://cdn.example.com/twitter.png">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/webp-og-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Modern Formats</title>
+  <meta property="og:image" content="https://cdn.example.com/hero.webp">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/helpers/HostQueue.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const HostQueue = require('@helpers/HostQueue')
+
+//
+// A controllable clock that also drives the queue's own timers, so advancing
+// time is instant and spacing assertions are exact rather than flaky.
+//
+const makeClock = () => {
+	let t = 1000
+	let nextId = 1
+	const timers = new Map()
+
+	return {
+		now: () => t,
+		setTimeoutFn: (fn, ms) => {
+			const id = nextId++
+			timers.set(id, { fn, dueAt: t + ms })
+			return id
+		},
+		clearTimeoutFn: (id) => { timers.delete(id) },
+		advance: (ms) => {
+			t += ms
+			for (const [id, timer] of [...timers.entries()]) {
+				if (timer.dueAt <= t) {
+					timers.delete(id)
+					timer.fn()
+				}
+			}
+		},
+	}
+}
+
+const wireClock = (clock) => ({
+	now: clock.now,
+	setTimeoutFn: clock.setTimeoutFn,
+	clearTimeoutFn: clock.clearTimeoutFn,
+})
+
+const deferred = () => {
+	let resolve = null
+	let reject = null
+	const promise = new Promise((res, rej) => { resolve = res; reject = rej })
+	return { promise, resolve, reject }
+}
+
+describe('global concurrency', () => {
+
+	test('never exceeds the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = Array.from({ length: 6 }, (unused, i) => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(`host${i}.com`, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+				return i
+			})
+		})
+
+		// Let the first batch start.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(2)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+
+		expect(peak).toBe(2)
+	})
+
+	test('runs different hosts in parallel up to the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = ['a.com', 'b.com', 'c.com'].map(host => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(host, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+			})
+		})
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(3)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+	})
+})
+
+describe('per-host serialisation', () => {
+
+	test('runs only one request per host at a time', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+
+		const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => {
+			active++
+			peak = Math.max(peak, active)
+			await new Promise(resolve => setImmediate(resolve))
+			active--
+		}))
+
+		await Promise.all(jobs)
+
+		expect(peak).toBe(1)
+	})
+
+	test('preserves FIFO order within a host', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+		const order = []
+
+		await Promise.all([1, 2, 3, 4].map(n =>
+			queue.schedule('same.com', async () => { order.push(n) })
+		))
+
+		expect(order).toEqual([1, 2, 3, 4])
+	})
+})
+
+describe('politeness spacing', () => {
+
+	test('leaves a gap between two requests to the same host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 1000,
+			maxSpacingMs: 1000,
+			...wireClock(clock),
+		})
+
+		const startedAt = []
+
+		const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+		await first
+
+		const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+
+		// The second job must not have run yet - the host is spaced out.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(startedAt).toHaveLength(1)
+
+		clock.advance(1000)
+		await second
+
+		expect(startedAt).toHaveLength(2)
+		expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000)
+	})
+
+	test('spacing on one host does not delay a different host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 5000,
+			maxSpacingMs: 5000,
+			...wireClock(clock),
+		})
+
+		await queue.schedule('slow.com', async () => {})
+
+		// slow.com is now parked for 5s. other.com must be unaffected.
+		const ran = []
+		await queue.schedule('other.com', async () => { ran.push('other') })
+
+		expect(ran).toEqual(['other'])
+	})
+
+	test('honours an explicit spacing override (robots Crawl-delay)', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) })
+
+		await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 })
+
+		const ran = []
+		const second = queue.schedule('slow.com', async () => { ran.push(1) })
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(ran).toEqual([])
+
+		clock.advance(3000)
+		await second
+		expect(ran).toEqual([1])
+	})
+})
+
+describe('error handling', () => {
+
+	test('a rejecting task does not wedge the queue', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', async () => { throw new Error('boom') }))
+			.rejects.toThrow('boom')
+
+		await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine')
+	})
+
+	test('a task that throws synchronously is caught', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', () => { throw new Error('sync boom') }))
+			.rejects.toThrow('sync boom')
+
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('one failing host does not block others', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const results = await Promise.allSettled([
+			queue.schedule('bad.com', async () => { throw new Error('nope') }),
+			queue.schedule('good.com', async () => 'ok'),
+		])
+
+		expect(results[0].status).toBe('rejected')
+		expect(results[1].value).toBe('ok')
+	})
+})
+
+describe('drain and stop', () => {
+
+	test('drain resolves when everything has finished', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+		const done = []
+
+		;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {}))
+
+		await queue.drain()
+
+		expect(done).toHaveLength(3)
+		expect(queue.size()).toBe(0)
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('drain resolves immediately on an idle queue', async () => {
+		const queue = new HostQueue.Queue()
+		await expect(queue.drain()).resolves.toBeUndefined()
+	})
+
+	test('stop rejects queued work and refuses new work', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const queued = queue.schedule('b.com', async () => 'never')
+
+		await new Promise(resolve => setImmediate(resolve))
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+		await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped')
+
+		gate.resolve('done')
+		await expect(first).resolves.toBe('done')
+	})
+})
+
+describe('event loop', () => {
+
+	//
+	// Regression: the politeness timer used to be unref'd, so if nothing else
+	// held the event loop open Node exited and queued jobs were abandoned with
+	// their promises never settling.
+	//
+	test('a spaced job still resolves with nothing else holding the loop open', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 })
+
+		await queue.schedule('same.com', async () => 'first')
+
+		// Only the internal timer can resume this one.
+		await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second')
+	})
+
+	test('stop() clears the pending timer so shutdown is not blocked', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 })
+
+		await queue.schedule('same.com', async () => 'first')
+		const queued = queue.schedule('same.com', async () => 'never')
+
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+	})
+})
+
+describe('stats', () => {
+
+	test('reports pending, running and host counts', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const second = queue.schedule('b.com', async () => {})
+
+		await new Promise(resolve => setImmediate(resolve))
+
+		const stats = queue.stats()
+		expect(stats.running).toBe(1)
+		expect(stats.pending).toBe(1)
+
+		gate.resolve()
+		await Promise.all([first, second])
+	})
+})

server/tests/helpers/HttpFetch.test.js

@@ -0,0 +1,479 @@
+require('module-alias/register')
+
+const HttpFetch = require('@helpers/HttpFetch')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+beforeEach(() => {
+	FetchMock.install()
+	// Every hostname in these tests resolves to one public address, so the
+	// safety gate passes and we are testing HttpFetch rather than DNS.
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+	// No real wall-time in backoff.
+	HttpFetch.__setSleep(() => Promise.resolve())
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	HttpFetch.__resetSleep()
+})
+
+describe('fetchDocument happy path', () => {
+
+	test('returns decoded html, status and final url', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<title>Hello</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.status).toBe(200)
+		expect(result.html).toContain('<title>Hello</title>')
+		expect(result.finalUrl).toBe('https://example.com/')
+		expect(result.notModified).toBe(false)
+	})
+
+	test('surfaces etag and last-modified for later conditional requests', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>', {
+			headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.etag).toBe('W/"abc"')
+		expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+})
+
+describe('fetchDocument browser identity', () => {
+
+	test('sends a complete, coherent header set and never claims to be Googlebot', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/')
+		const headers = FetchMock.calls()[0].headers
+
+		expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/)
+		expect(headers['User-Agent']).not.toMatch(/Googlebot/i)
+		expect(headers['Accept']).toContain('text/html')
+		expect(headers['Accept-Language']).toBeTruthy()
+		expect(headers['Accept-Encoding']).toContain('gzip')
+	})
+
+	test('gives the same host the same profile across calls', async () => {
+		FetchMock.route(/example\.com/, () => FetchMock.html('<p>x</p>'))
+
+		const first = await HttpFetch.fetchDocument('https://example.com/a')
+		const second = await HttpFetch.fetchDocument('https://example.com/b')
+
+		expect(first.profileId).toBe(second.profileId)
+	})
+
+	test('pairs sec-ch-ua only with Chromium profiles', () => {
+		const chrome = BrowserProfiles.profileById('chrome-win')
+		const firefox = BrowserProfiles.profileById('firefox-win')
+		const safari = BrowserProfiles.profileById('safari-mac')
+
+		const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' })
+		const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' })
+		const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' })
+
+		expect(chromeHeaders['sec-ch-ua']).toBeTruthy()
+		expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Firefox sends Sec-Fetch-* but no client hints.
+		expect(firefoxHeaders['sec-ch-ua']).toBeUndefined()
+		expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Safari sends neither.
+		expect(safariHeaders['sec-ch-ua']).toBeUndefined()
+		expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined()
+	})
+
+	test('never pairs a chrome UA with a firefox accept header', () => {
+		BrowserProfiles.PROFILES.forEach(profile => {
+			if (profile.family === 'chrome') {
+				expect(profile.ua).toMatch(/Chrome\//)
+				expect(profile.accept).toContain('application/signed-exchange')
+			}
+			if (profile.family === 'firefox') {
+				expect(profile.ua).toMatch(/Firefox\//)
+				expect(profile.accept).not.toContain('signed-exchange')
+			}
+		})
+	})
+})
+
+describe('fetchDocument redirects', () => {
+
+	test('follows a single hop and reports the final url', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301))
+		FetchMock.route('https://example.com/b', () => FetchMock.html('<title>B</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+
+		expect(result.finalUrl).toBe('https://example.com/b')
+		expect(result.html).toContain('B')
+	})
+
+	test('resolves a relative Location header', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('/c', 302))
+		FetchMock.route('https://example.com/c', () => FetchMock.html('<title>C</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+		expect(result.finalUrl).toBe('https://example.com/c')
+	})
+
+	test('detects a redirect loop', async () => {
+		FetchMock.route('https://example.com/a', () => FetchMock.redirect('https://example.com/b', 302))
+		FetchMock.route('https://example.com/b', () => FetchMock.redirect('https://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'redirect_loop' })
+	})
+
+	test('gives up past the redirect cap', async () => {
+		let n = 0
+		FetchMock.route(/https:\/\/example\.com\/hop/, () => FetchMock.redirect(`https://example.com/hop${++n}`, 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/hop'))
+			.rejects.toMatchObject({ kind: 'too_many_redirects' })
+	})
+
+	// The SSRF case that matters: the first URL is perfectly public.
+	test('blocks a redirect pointing at loopback', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://127.0.0.1:3000/admin', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('blocks a redirect pointing at the cloud metadata endpoint', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://169.254.169.254/latest/meta-data/', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('refuses an https to http downgrade', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+})
+
+describe('fetchDocument retries', () => {
+
+	test('retries a 500 and succeeds', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(500),
+			FetchMock.html('<title>Recovered</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('Recovered')
+		expect(FetchMock.callCount()).toBe(2)
+	})
+
+	test('gives up after exhausting retries', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(503))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 503 })
+
+		// 1 initial attempt + config.retries
+		expect(FetchMock.callCount()).toBe(3)
+	})
+
+	test('does NOT retry a 404 - it is an answer, not a failure', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('does NOT retry a 403', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(403))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/')).rejects.toMatchObject({ status: 403 })
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('honours a short Retry-After on a 429', async () => {
+		const waits = []
+		HttpFetch.__setSleep((ms) => { waits.push(ms); return Promise.resolve() })
+
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(429, { headers: { 'retry-after': '2' } }),
+			FetchMock.html('<title>OK</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('OK')
+		expect(waits[0]).toBe(2000)
+	})
+
+	test('refuses to sit on an hour-long Retry-After', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(429, { headers: { 'retry-after': '3600' } }))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 429 })
+
+		// Surfaced for the cache to park the URL rather than retried.
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('retries a network error', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.networkError('ECONNRESET'),
+			FetchMock.html('<title>Up</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('Up')
+	})
+})
+
+describe('fetchDocument limits and gates', () => {
+
+	test('times out on a server that never responds', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.hang())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { timeoutMs: 60, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'timeout' })
+	})
+
+	test('aborts a body that exceeds the cap mid-stream', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.oversized())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { maxBytes: 128 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects an oversized declared content-length without reading the body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>small</p>', {
+			headers: { 'content-length': String(50 * 1024 * 1024) },
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('refuses a non-html content type before parsing', async () => {
+		FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'application/pdf',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('refuses a binary masquerading behind an image content type', async () => {
+		FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'image/png',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+})
+
+describe('fetchDocument charset handling', () => {
+
+	test('decodes windows-1252 declared in the content-type header', async () => {
+		// 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8.
+		const body = Buffer.concat([
+			Buffer.from('<title>', 'latin1'),
+			Buffer.from([0x93]),
+			Buffer.from('caf', 'latin1'),
+			Buffer.from([0xe9, 0x94]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=windows-1252',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('“café”')
+		expect(result.charset).toBe('windows-1252')
+	})
+
+	test('falls back to a meta charset when the header says nothing', async () => {
+		const body = Buffer.concat([
+			Buffer.from('<meta charset="windows-1252"><title>caf', 'latin1'),
+			Buffer.from([0xe9]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, { contentType: 'text/html' }))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('header charset wins over the meta tag', async () => {
+		const body = Buffer.from('<meta charset="windows-1252"><title>café</title>', 'utf8')
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=utf-8',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('survives a bogus charset label', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(Buffer.from('<p>ok</p>'), {
+			contentType: 'text/html; charset=not-a-real-charset',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('ok')
+	})
+})
+
+describe('fetchDocument conditional requests', () => {
+
+	test('sends If-None-Match and If-Modified-Since when given', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		})
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['If-None-Match']).toBe('W/"v1"')
+		expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+
+	test('reports a 304 as notModified with no body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(304))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"' },
+		})
+
+		expect(result.notModified).toBe(true)
+		expect(result.status).toBe(304)
+		expect(result.html).toBeNull()
+	})
+})
+
+describe('fetchBinary', () => {
+
+	const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+
+	test('returns the buffer for a real image', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		const result = await HttpFetch.fetchBinary('https://example.com/i.png')
+
+		expect(result.mime).toBe('image/png')
+		expect(result.buffer.equals(pngMagic)).toBe(true)
+	})
+
+	// The old code wrote a 404 HTML error page to disk and handed it to gm.
+	test('refuses an html error page served where an image was expected', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.html('<h1>404</h1>'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('propagates a 404 status rather than writing anything', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+	})
+
+	test('enforces the image size cap', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.oversized('image/png'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { maxBytes: 64 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects svg by default', async () => {
+		FetchMock.route('https://example.com/i.svg', () => FetchMock.buffer(Buffer.from('<svg/>'), {
+			contentType: 'image/svg+xml',
+		}))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.svg', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('sends image-appropriate fetch metadata and a page referer', async () => {
+		FetchMock.route('https://cdn.example.net/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		await HttpFetch.fetchBinary('https://cdn.example.net/i.png', { pageUrl: 'https://example.com/post' })
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['Referer']).toBe('https://example.com/post')
+		expect(headers['Accept']).toContain('image/')
+		if (headers['Sec-Fetch-Dest']) {
+			expect(headers['Sec-Fetch-Dest']).toBe('image')
+			expect(headers['Sec-Fetch-Site']).toBe('cross-site')
+		}
+	})
+})
+
+describe('transient vs terminal safety failures', () => {
+
+	// A private address is a permanent refusal; a DNS failure means the host
+	// may just be down. Classifying both as "blocked" parked transient
+	// failures for the full cache TTL.
+	test('classifies a private address as blocked', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' })
+	})
+
+	test('classifies a DNS failure as network, not blocked', async () => {
+		dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' })
+	})
+
+	test('retries a DNS failure, since it may be transient', async () => {
+		let attempts = 0
+		dnsSpy.mockImplementation(async () => {
+			attempts++
+			if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) }
+			return [{ address: '93.184.216.34', family: 4 }]
+		})
+		FetchMock.route('https://flaky.example/', () => FetchMock.html('<title>Back up</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://flaky.example/')
+		expect(result.html).toContain('Back up')
+		expect(attempts).toBe(2)
+	})
+
+	test('does NOT retry a private address', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' })
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})
+
+describe('safety gate on the initial url', () => {
+
+	test('refuses to open a connection to a private address at all', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>should never be reached</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked' })
+
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})

server/tests/helpers/Robots.test.js

@@ -0,0 +1,173 @@
+require('module-alias/register')
+
+const Robots = require('@helpers/Robots')
+const config = require('@config/scrape')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+const UA = config.robotsUaToken
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	Robots.__resetCache()
+})
+
+const serveRobots = (body, origin = 'https://example.com') => {
+	FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
+		status: 200,
+		headers: { 'content-type': 'text/plain' },
+	}))
+}
+
+describe('Robots.parse', () => {
+
+	test('groups consecutive user-agent lines into one rule block', () => {
+		const rules = Robots.parse([
+			'User-agent: alpha',
+			'User-agent: beta',
+			'Disallow: /private',
+		].join('\n'))
+
+		expect(rules.groups).toHaveLength(1)
+		expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
+	})
+
+	test('ignores comments and blank lines', () => {
+		const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
+	})
+
+	test('treats an empty Disallow as no restriction', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow:')
+		expect(rules.groups[0].rules).toHaveLength(0)
+	})
+
+	test('parses crawl-delay', () => {
+		const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
+		expect(rules.groups[0].crawlDelayMs).toBe(2500)
+	})
+
+	test('an empty file allows everything', () => {
+		expect(Robots.parse('').allowAll).toBe(true)
+		expect(Robots.parse(null).allowAll).toBe(true)
+	})
+})
+
+describe('Robots.isPathAllowed', () => {
+
+	test('respects a simple disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /private')
+		expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
+	})
+
+	test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
+		expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
+	})
+
+	test('Allow beats Disallow at equal specificity', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
+		expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
+	})
+
+	test('supports the * wildcard', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
+		expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
+	})
+
+	test('supports the $ anchor', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /x$')
+		expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
+	})
+
+	test('a named group beats the wildcard group even when listed later', () => {
+		const rules = Robots.parse([
+			'User-agent: *',
+			'Disallow: /',
+			'',
+			`User-agent: ${UA}`,
+			'Disallow:',
+		].join('\n'))
+
+		expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
+		expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
+	})
+})
+
+describe('Robots.isAllowed over the network', () => {
+
+	test('refuses a disallowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
+	})
+
+	test('permits an allowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
+	})
+
+	// Fail-open: a missing robots.txt is not a refusal.
+	test('allows everything when robots.txt 404s', async () => {
+		FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('allows everything when robots.txt cannot be reached', async () => {
+		FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('caches per origin - two queries, one network call', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+
+		await Robots.isAllowed('https://example.com/a')
+		await Robots.isAllowed('https://example.com/secret/b')
+
+		expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
+	})
+
+	test('keeps origins separate', async () => {
+		serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
+		serveRobots('User-agent: *\nDisallow:', 'https://open.com')
+
+		await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
+		await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
+	})
+
+	test('returns false for an unparseable url rather than throwing', async () => {
+		await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
+	})
+})
+
+describe('Robots.crawlDelayMs', () => {
+
+	test('reports the declared delay', async () => {
+		serveRobots('User-agent: *\nCrawl-delay: 5')
+		await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
+	})
+
+	test('is null when none is declared', async () => {
+		serveRobots('User-agent: *\nDisallow: /x')
+		await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
+	})
+
+	test('flags a delay beyond the cap instead of parking the job forever', async () => {
+		expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
+		expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
+		expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
+	})
+})

server/tests/helpers/SiteScrape.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const fs = require('fs')
+const path = require('path')
+const cheerio = require('cheerio')
+
+const SiteScrape = require('@helpers/SiteScrape')
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')
+
+// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
+const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))
+
+const PAGE_URL = 'https://example.com/blog/2026/post'
+
+describe('getTitle', () => {
+
+	test('prefers og:title over the title element', () => {
+		expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
+	})
+
+	test('falls back to twitter:title', () => {
+		expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
+	})
+
+	test('falls back to JSON-LD headline', () => {
+		expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
+	})
+
+	test('falls back to the title element and collapses whitespace', () => {
+		expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
+	})
+
+	test('falls back to an h1 when there is no title element', () => {
+		expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
+	})
+
+	test('returns empty string for a page with nothing', () => {
+		expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
+	})
+})
+
+describe('getDescription', () => {
+
+	test('prefers og:description', () => {
+		expect(SiteScrape.getDescription(load('og-complete.html')))
+			.toBe('A proper social description of the page.')
+	})
+
+	test('falls back to twitter:description', () => {
+		expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
+	})
+
+	test('falls back to JSON-LD description', () => {
+		expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
+	})
+
+	test('is empty when no description exists', () => {
+		expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
+	})
+})
+
+describe('getJsonLd', () => {
+
+	test('reads a plain block', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
+	})
+
+	test('digs the content node out of an @graph', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
+	})
+
+	// Malformed JSON-LD is extremely common and must never throw.
+	test('survives malformed JSON without throwing', () => {
+		expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
+		expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
+	})
+
+	test('is null when absent', () => {
+		expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
+	})
+})
+
+describe('getCanonicalUrl', () => {
+
+	test('uses link rel=canonical', () => {
+		expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
+			.toBe('https://example.com/canonical-post')
+	})
+
+	test('falls back to the requested url', () => {
+		expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
+	})
+})
+
+describe('getDisplayImage', () => {
+
+	test('prefers og:image', () => {
+		expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.jpg')
+	})
+
+	test('falls back to twitter:image', () => {
+		expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/twitter.png')
+	})
+
+	test('falls back to the JSON-LD image, taking the first of an array', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/ld-primary.jpg')
+	})
+
+	test('reads an object-form JSON-LD image', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/graph.png')
+	})
+
+	// Resolution against the canonical base, not string concatenation.
+	test('resolves a relative path against the canonical url', () => {
+		expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
+			.toBe('https://example.com/blog/images/hero.png')
+	})
+
+	test('resolves a root-relative icon', () => {
+		expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
+			.toBe('https://example.com/favicon.ico')
+	})
+
+	test('resolves a protocol-relative url', () => {
+		expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/proto.jpg')
+	})
+
+	// The old substring filter dropped all of these.
+	test('accepts .webp', () => {
+		expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.webp')
+	})
+
+	test('accepts an extensionless og:image endpoint', () => {
+		expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
+			.toBe('https://example.com/api/og?title=Hello&theme=dark')
+	})
+
+	test('rejects svg while allowSvgThumbnails is off', () => {
+		expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
+	})
+
+	test('picks the largest srcset candidate', () => {
+		expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/large.jpg')
+	})
+
+	test('prefers the largest inline image by declared area', () => {
+		expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/big.jpg')
+	})
+
+	test('skips tracking pixels and spacers', () => {
+		expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/real-photo.jpg')
+	})
+
+	// Regression: the old code read element.attribs.src directly and threw.
+	test('does not throw on img tags with no src, and finds a data-src', () => {
+		const $ = load('imgs-missing-src.html')
+		expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
+		expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
+	})
+
+	test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
+		expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
+			.toBe('https://example.com/touch-180.png')
+	})
+
+	test('returns empty string when there is no usable image', () => {
+		expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
+	})
+
+	test('survives malformed markup', () => {
+		expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
+	})
+})
+
+describe('getImageCandidates', () => {
+
+	test('returns an ordered, deduplicated list', () => {
+		const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
+		expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
+	})
+
+	test('is empty rather than null for a bare page', () => {
+		expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
+	})
+})
+
+describe('getKeywords', () => {
+
+	test('extracts page terms and ignores nav, script and footer chrome', () => {
+		const keywords = SiteScrape.getKeywords(load('keyword-content.html'))
+
+		expect(keywords).toMatch(/^Keywords: /)
+		expect(keywords).toMatch(/beekeeping|beehive|honey/)
+		expect(keywords).not.toMatch(/navigation/)
+		expect(keywords).not.toMatch(/tracking/)
+		expect(keywords).not.toMatch(/footer/)
+	})
+
+	test('returns empty string when there is nothing to count', () => {
+		expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
+	})
+
+	test('does not log page content', () => {
+		// The old implementation dumped up to 6000 chars of every page.
+		const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
+		SiteScrape.getKeywords(load('keyword-content.html'))
+		expect(spy).not.toHaveBeenCalled()
+		spy.mockRestore()
+	})
+})
+
+describe('getHostName', () => {
+
+	test('returns the https origin', () => {
+		expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
+	})
+
+	test('returns empty string for a bad url instead of throwing', () => {
+		expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
+		expect(SiteScrape.getHostName('nonsense')).toBe('')
+	})
+})
+
+describe('buildSearchText', () => {
+
+	test('joins the parts that exist', () => {
+		const text = SiteScrape.buildSearchText({
+			title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
+		})
+		expect(text).toBe('T\nD\nS\nKeywords: a, b')
+	})
+
+	test('skips empty parts', () => {
+		expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
+	})
+
+	test('caps the length so it cannot bloat attachment.text', () => {
+		const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
+		expect(text.length).toBeLessThanOrEqual(8000)
+	})
+
+	// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
+	test('never produces the old failure sentinels', () => {
+		const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
+		expect(text).toBe('')
+		expect(text).not.toContain('No site text')
+		expect(text).not.toContain('Request Timeout')
+	})
+})
+
+describe('parse', () => {
+
+	test('returns everything a link preview needs in one call', () => {
+		const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)
+
+		expect(result).toMatchObject({
+			title: 'The Real Title',
+			description: 'A proper social description of the page.',
+			siteName: 'Example Journal',
+			canonicalUrl: 'https://example.com/canonical-post',
+			imageUrl: 'https://cdn.example.com/hero.jpg',
+			hostname: 'https://example.com',
+		})
+		expect(result.searchText).toContain('The Real Title')
+		expect(result.searchText).toContain('A proper social description')
+	})
+
+	test('degrades to empty fields rather than throwing on a bare page', () => {
+		const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)
+
+		expect(result.title).toBe('')
+		expect(result.imageUrl).toBe('')
+		expect(result.searchText).toBe('example.com')
+	})
+})
+
+//
+// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
+// note text and what string is stored in attachment.url. These tests pin the
+// CURRENT behaviour so the rewrite around it cannot drift link detection.
+//
+describe('getCleanUrls (pinned, must not change)', () => {
+
+	test('finds bare and full urls', () => {
+		expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
+			.toEqual(['https://example.com/a', 'https://www.foo.com'])
+	})
+
+	test('still forces http to https', () => {
+		expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
+	})
+
+	test('still prepends https to a bare www host', () => {
+		expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
+	})
+
+	test('deduplicates', () => {
+		expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
+	})
+
+	test('returns an empty array when there are no urls', () => {
+		expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
+	})
+
+	test('returns an empty array for empty text', () => {
+		expect(SiteScrape.getCleanUrls('')).toEqual([])
+	})
+})

server/tests/helpers/UrlSafety.test.js

@@ -0,0 +1,189 @@
+require('module-alias/register')
+
+const UrlSafety = require('@helpers/UrlSafety')
+
+describe('UrlSafety.isPrivateIp', () => {
+
+	const privateAddresses = [
+		'0.0.0.0', '10.0.0.1', '10.255.255.254', '127.0.0.1', '127.1.1.1',
+		'169.254.1.1', '169.254.169.254',  // cloud metadata endpoint
+		'172.16.0.1', '172.31.255.254', '192.168.1.1', '192.0.0.1', '192.0.2.5',
+		'198.18.0.1', '100.64.0.1', '100.127.255.254', '224.0.0.1', '255.255.255.255',
+		'::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1',
+		'::ffff:127.0.0.1', '::ffff:10.0.0.1', '64:ff9b::127.0.0.1',
+	]
+
+	privateAddresses.forEach(address => {
+		test(`blocks ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(true)
+		})
+	})
+
+	const publicAddresses = ['1.1.1.1', '8.8.8.8', '93.184.216.34', '172.32.0.1', '2606:4700::1111']
+
+	publicAddresses.forEach(address => {
+		test(`allows ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(false)
+		})
+	})
+
+	test('treats a non-IP string as unsafe', () => {
+		expect(UrlSafety.isPrivateIp('not-an-ip')).toBe(true)
+		expect(UrlSafety.isPrivateIp('')).toBe(true)
+	})
+})
+
+describe('UrlSafety.parse', () => {
+
+	test('rejects non-http schemes', () => {
+		expect(UrlSafety.parse('file:///etc/passwd')).toBeNull()
+		expect(UrlSafety.parse('ftp://example.com/x')).toBeNull()
+		expect(UrlSafety.parse('javascript:alert(1)')).toBeNull()
+		expect(UrlSafety.parse('data:text/html,<h1>x</h1>')).toBeNull()
+	})
+
+	test('rejects garbage and empty input', () => {
+		expect(UrlSafety.parse('')).toBeNull()
+		expect(UrlSafety.parse(null)).toBeNull()
+		expect(UrlSafety.parse('not a url at all')).toBeNull()
+	})
+
+	test('rejects URLs longer than the attachment.url column', () => {
+		const long = 'https://example.com/' + 'a'.repeat(1200)
+		expect(UrlSafety.parse(long)).toBeNull()
+	})
+
+	test('accepts ordinary http and https', () => {
+		expect(UrlSafety.parse('https://example.com/a').href).toBe('https://example.com/a')
+		expect(UrlSafety.parse('http://example.com/a').href).toBe('http://example.com/a')
+	})
+})
+
+describe('UrlSafety.normalizeForKey / urlKey', () => {
+
+	test('strips tracking params so the cache key is stable', () => {
+		const bare = UrlSafety.urlKey('https://example.com/post')
+		const noisy = UrlSafety.urlKey('https://example.com/post?utm_source=x&utm_medium=y&fbclid=z')
+		expect(noisy).toBe(bare)
+	})
+
+	test('keeps meaningful query params', () => {
+		const a = UrlSafety.urlKey('https://example.com/search?q=cats')
+		const b = UrlSafety.urlKey('https://example.com/search?q=dogs')
+		expect(a).not.toBe(b)
+	})
+
+	test('is insensitive to param order, host case, fragment and trailing host dot', () => {
+		const canonical = UrlSafety.urlKey('https://example.com/p?a=1&b=2')
+		expect(UrlSafety.urlKey('https://example.com/p?b=2&a=1')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://EXAMPLE.com/p?a=1&b=2')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com/p?a=1&b=2#section')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com./p?a=1&b=2')).toBe(canonical)
+	})
+
+	test('distinguishes http from https', () => {
+		expect(UrlSafety.urlKey('http://example.com/')).not.toBe(UrlSafety.urlKey('https://example.com/'))
+	})
+
+	test('returns a 64-char hex digest', () => {
+		expect(UrlSafety.urlKey('https://example.com/')).toMatch(/^[0-9a-f]{64}$/)
+	})
+
+	test('returns null for an unusable URL', () => {
+		expect(UrlSafety.urlKey('javascript:alert(1)')).toBeNull()
+	})
+})
+
+describe('UrlSafety.sameResource', () => {
+
+	// This is what stops scanTextForWebsites deleting and re-scraping a row
+	// every time a note is saved.
+	test('treats http/https, www. and trailing slash as the same resource', () => {
+		expect(UrlSafety.sameResource('http://foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://www.foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com/a/', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com', 'https://foo.com/')).toBe(true)
+	})
+
+	test('does not conflate different paths, hosts or queries', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://foo.com/b')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://bar.com/a')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a?x=1', 'https://foo.com/a?x=2')).toBe(false)
+	})
+
+	test('is false when either side is unusable', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'javascript:x')).toBe(false)
+	})
+})
+
+describe('UrlSafety.assertFetchable', () => {
+
+	const expectRejection = async (url, reason) => {
+		await expect(UrlSafety.assertFetchable(url)).rejects.toMatchObject({ name: 'SafetyError', reason })
+	}
+
+	test('refuses non-http schemes', async () => {
+		await expectRejection('file:///etc/passwd', 'bad_url')
+	})
+
+	test('refuses embedded credentials', async () => {
+		await expectRejection('https://user:[email protected]/', 'credentials_in_url')
+	})
+
+	test('refuses localhost and internal suffixes', async () => {
+		await expectRejection('http://localhost:3000/', 'blocked_host')
+		await expectRejection('http://db.internal/', 'blocked_host')
+		await expectRejection('http://thing.local/', 'blocked_host')
+		await expectRejection('http://box.lan/', 'blocked_host')
+	})
+
+	test('refuses literal private addresses without touching DNS', async () => {
+		await expectRejection('http://127.0.0.1:3000/', 'private_ip')
+		await expectRejection('http://169.254.169.254/latest/meta-data/', 'private_ip')
+		await expectRejection('http://10.1.2.3/', 'private_ip')
+		await expectRejection('http://[::1]:8080/', 'private_ip')
+	})
+
+	test('allows a literal public address', async () => {
+		await expect(UrlSafety.assertFetchable('http://1.1.1.1/')).resolves.toEqual(['1.1.1.1'])
+	})
+
+	test('refuses a hostname that resolves to a private address', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '192.168.0.5', family: 4 }])
+
+		await expectRejection('https://rebind.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('refuses when ANY address in a round-robin record is private', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([
+			{ address: '93.184.216.34', family: 4 },
+			{ address: '127.0.0.1', family: 4 },
+		])
+
+		await expectRejection('https://mixed.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('allows a hostname that resolves entirely to public addresses', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+
+		await expect(UrlSafety.assertFetchable('https://example.com/')).resolves.toEqual(['93.184.216.34'])
+
+		spy.mockRestore()
+	})
+
+	test('surfaces a DNS failure as dns_failed', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+
+		await expectRejection('https://nonexistent.example/', 'dns_failed')
+
+		spy.mockRestore()
+	})
+})

server/tests/models/ScrapeQueue.test.js

@@ -0,0 +1,354 @@
+require('module-alias/register')
+
+const db = require('@config/database')
+const User = require('@models/User')
+const Attachment = require('@models/Attachment')
+const ScrapeCache = require('@models/ScrapeCache')
+const ScrapeQueue = require('@workers/ScrapeQueue')
+const Schema = require('@helpers/Schema')
+const Robots = require('@helpers/Robots')
+const HttpFetch = require('@helpers/HttpFetch')
+const UrlSafety = require('@helpers/UrlSafety')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+
+const testUserName = 'jestTestUserScrapeQueue'
+const password = 'Beans19934!!!'
+
+let userId = null
+let dnsSpy = null
+
+const page = (title, description) => FetchMock.html(`
+	<html><head>
+		<title>${title}</title>
+		<meta property="og:description" content="${description || 'a description'}">
+	</head><body><p>body</p></body></html>
+`)
+
+// Clear cache rows for a URL so each test starts from a real miss.
+const clearCache = (url) => db.promise()
+	.query('DELETE FROM scrape_cache WHERE url_hash = ?', [UrlSafety.urlKey(url)])
+
+const attachmentRow = (id) => db.promise()
+	.query('SELECT * FROM attachment WHERE id = ?', [id])
+	.then(([rows]) => rows[0] || null)
+
+beforeAll(async () => {
+	await Schema.ensureScrapeSchema()
+
+	try {
+		const existing = await User.getByUserName(testUserName)
+		if (existing && existing.id) { await User.deleteUser(existing.id, password) }
+	} catch (error) { /* first run */ }
+
+	const registered = await User.register(testUserName, password)
+	userId = registered.userId
+})
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+	HttpFetch.__setSleep(() => Promise.resolve())
+
+	// Exercise the real code path with politeness spacing and robots disabled.
+	ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	HttpFetch.__resetSleep()
+	ScrapeQueue.__reset()
+})
+
+afterAll(async () => {
+	await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
+	db.end()
+})
+
+describe('fetch and persist', () => {
+
+	test('writes parsed metadata onto the attachment row as plaintext', async () => {
+		const url = 'https://example.com/post-one'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Post One', 'the first post'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+
+		expect(row.text).toContain('Post One')
+		expect(row.text).toContain('the first post')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+		// Must stay unencrypted - the FULLTEXT index depends on it.
+		expect(row.text).not.toMatch(/^[0-9a-f]{64}/)
+	})
+
+	test('keeps the url the user wrote rather than the canonical url', async () => {
+		const url = 'https://example.com/original'
+		await clearCache(url)
+
+		FetchMock.route(url, () => FetchMock.html(`
+			<html><head><title>Canonical Test</title>
+			<link rel="canonical" href="https://example.com/different-canonical">
+			</head><body>x</body></html>
+		`))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+		expect(row.url).toBe(url)
+
+		// The canonical URL is recorded in the cache, not on the attachment.
+		const cached = await ScrapeCache.get(url)
+		expect(cached.canonical_url).toBe('https://example.com/different-canonical')
+	})
+})
+
+describe('caching', () => {
+
+	test('a second request for the same url makes zero network calls', async () => {
+		const url = 'https://example.com/cached'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Cached Page'))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		const callsAfterFirst = FetchMock.callCount()
+		expect(callsAfterFirst).toBeGreaterThan(0)
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		expect(FetchMock.callCount()).toBe(callsAfterFirst)
+
+		// The cached row still populates the second attachment.
+		const row = await attachmentRow(secondId)
+		expect(row.text).toContain('Cached Page')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('two notes sharing a url coalesce into one fetch', async () => {
+		const url = 'https://example.com/shared'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Shared Page'))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+
+		// Enqueued together, so they overlap in flight.
+		await ScrapeQueue.enqueueUrls([
+			{ userId, noteId: null, attachmentId: firstId, url },
+			{ userId, noteId: null, attachmentId: secondId, url },
+		])
+
+		expect(FetchMock.callsTo(url)).toHaveLength(1)
+
+		const rows = await Promise.all([attachmentRow(firstId), attachmentRow(secondId)])
+		rows.forEach(row => {
+			expect(row.text).toContain('Shared Page')
+			expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+		})
+	})
+
+	test('a stale entry revalidates and a 304 reuses the stored text', async () => {
+		const url = 'https://example.com/revalidate'
+		await clearCache(url)
+
+		FetchMock.routeSequence(url, [
+			FetchMock.html('<html><head><title>Original Title</title></head><body>x</body></html>', {
+				headers: { etag: 'W/"v1"' },
+			}),
+			FetchMock.status(304),
+		])
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		// Force staleness so the next pass revalidates instead of hitting cache.
+		await db.promise().query('UPDATE scrape_cache SET expires_at = ? WHERE url_hash = ?',
+			[Math.round(Date.now() / 1000) - 10, UrlSafety.urlKey(url)])
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		// Conditional request was sent.
+		const conditional = FetchMock.callsTo(url).find(call => call.headers['If-None-Match'])
+		expect(conditional).toBeTruthy()
+
+		const row = await attachmentRow(secondId)
+		expect(row.text).toContain('Original Title')
+	})
+})
+
+describe('failure handling', () => {
+
+	test('a 500 leaves the row failed and never writes a sentinel string', async () => {
+		const url = 'https://example.com/broken'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(500))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.FAILED)
+		// The old code stored these in attachment.text and fed them to FULLTEXT.
+		expect(row.text).not.toContain('No site text')
+		expect(row.text).not.toContain('Request Timeout')
+	})
+
+	test('a 404 removes the shell row rather than orphaning it', async () => {
+		const url = 'https://example.com/gone'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(404))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+	})
+
+	test('a blocked url removes the shell row and makes no request', async () => {
+		const url = 'http://127.0.0.1:9999/internal'
+		FetchMock.route(/.*/, () => page('should never load'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('one failing url does not strand the rest of a batch', async () => {
+		const good = 'https://example.com/good-one'
+		const bad = 'https://example.com/bad-one'
+		await clearCache(good)
+		await clearCache(bad)
+
+		FetchMock.route(good, () => page('Good One'))
+		FetchMock.route(bad, () => FetchMock.status(500))
+
+		const goodId = await Attachment.createPendingUrlAttachment(userId, null, good)
+		const badId = await Attachment.createPendingUrlAttachment(userId, null, bad)
+
+		// The old counter-based implementation never settled in this case.
+		const texts = await ScrapeQueue.enqueueUrls([
+			{ userId, noteId: null, attachmentId: badId, url: bad },
+			{ userId, noteId: null, attachmentId: goodId, url: good },
+		])
+
+		expect(texts).toHaveLength(2)
+		expect((await attachmentRow(goodId)).text).toContain('Good One')
+	})
+
+	test('a failed url is not refetched while inside its backoff window', async () => {
+		const url = 'https://example.com/flaky'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(500))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		const callsAfterFirst = FetchMock.callCount()
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		expect(FetchMock.callCount()).toBe(callsAfterFirst)
+	})
+})
+
+describe('robots.txt', () => {
+
+	test('refuses a disallowed url and records it', async () => {
+		const url = 'https://example.com/no-bots/page'
+		await clearCache(url)
+
+		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 }) // robots ON
+
+		FetchMock.route('https://example.com/robots.txt', () => new Response(
+			'User-agent: *\nDisallow: /no-bots', { status: 200, headers: { 'content-type': 'text/plain' } }
+		))
+		FetchMock.route(url, () => page('Should Not Be Fetched'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(FetchMock.callsTo(url)).toHaveLength(0)
+
+		const cached = await ScrapeCache.get(url)
+		expect(cached.fetch_state).toBe('blocked')
+	})
+
+	test('proceeds when robots.txt allows the path', async () => {
+		const url = 'https://example.com/allowed/page'
+		await clearCache(url)
+
+		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 })
+
+		FetchMock.route('https://example.com/robots.txt', () => new Response(
+			'User-agent: *\nDisallow: /private', { status: 200, headers: { 'content-type': 'text/plain' } }
+		))
+		FetchMock.route(url, () => page('Allowed Page'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect((await attachmentRow(attachmentId)).text).toContain('Allowed Page')
+	})
+})
+
+describe('crash recovery', () => {
+
+	test('re-enqueues rows left pending by a restart', async () => {
+		const url = 'https://example.com/recovered'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Recovered Page'))
+
+		// Simulate a row that was enqueued but never processed.
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+
+		const recovery = await ScrapeQueue.recoverPending()
+		expect(recovery.requeued).toBeGreaterThan(0)
+
+		await ScrapeQueue.drain()
+
+		const row = await attachmentRow(attachmentId)
+		expect(row.text).toContain('Recovered Page')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('resets rows stuck mid-fetch so they are retried', async () => {
+		const url = 'https://example.com/stuck'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Unstuck Page'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		// Pretend the process died while this one was in flight.
+		await db.promise().query('UPDATE attachment SET scrape_state = ? WHERE id = ?',
+			[ScrapeQueue.STATE.FETCHING, attachmentId])
+
+		await ScrapeQueue.recoverPending()
+		await ScrapeQueue.drain()
+
+		expect((await attachmentRow(attachmentId)).scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('does not sweep up long-settled rows', async () => {
+		// Rows default to DONE, which is what keeps the pre-existing ~1400
+		// attachment rows out of the recovery queue on first boot.
+		const [rows] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE scrape_state = ? AND user_id = ?',
+			[ScrapeQueue.STATE.PENDING, userId]
+		)
+		const recovery = await ScrapeQueue.recoverPending()
+		expect(recovery.requeued).toBeLessThanOrEqual(rows[0].n + 1)
+	})
+})

server/tests/support/fetchMock.js

@@ -0,0 +1,127 @@
+//
+// Test harness for the outbound HTTP layer.
+//
+// Responses are built from the REAL Response/ReadableStream globals rather than
+// hand-rolled stubs, so the streaming size cap, the charset decoder and the
+// body-cancel paths in HttpFetch are genuinely exercised instead of being
+// stubbed around.
+//
+
+let FetchMock = module.exports = {}
+
+let originalFetch = null
+let routes = []
+let callLog = []
+
+FetchMock.install = () => {
+	originalFetch = global.fetch
+	routes = []
+	callLog = []
+
+	global.fetch = jest.fn(async (url, options = {}) => {
+		const href = typeof url === 'string' ? url : url.href
+		callLog.push({ url: href, headers: options.headers || {}, options })
+
+		const route = routes.find(entry => entry.match(href))
+		if (!route) {
+			throw Object.assign(new Error(`No route for ${href}`), { code: 'ENOTFOUND' })
+		}
+
+		// A hanging route must still honour the abort signal, or the test would
+		// wait forever instead of proving the AbortController fires.
+		if (options.signal) {
+			return await Promise.race([
+				Promise.resolve(route.responder(href, options)),
+				abortPromise(options.signal),
+			])
+		}
+
+		return route.responder(href, options)
+	})
+}
+
+const abortPromise = (signal) => new Promise((resolve, reject) => {
+	if (signal.aborted) {
+		return reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
+	}
+	signal.addEventListener('abort', () => {
+		reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
+	}, { once: true })
+})
+
+FetchMock.restore = () => {
+	if (originalFetch) { global.fetch = originalFetch } else { delete global.fetch }
+	originalFetch = null
+	routes = []
+	callLog = []
+}
+
+//
+// route('https://example.com/', responder) or route(/example/, responder).
+// Responders are called with (url, options) and may be functions returning a
+// Response, so a route can answer differently on each call.
+//
+FetchMock.route = (pattern, responder) => {
+	const match = pattern instanceof RegExp
+		? (href) => pattern.test(href)
+		: (href) => href === pattern
+
+	routes.push({ match, responder: typeof responder === 'function' ? responder : () => responder })
+}
+
+// Answer with a counter so "retried exactly N times" is assertable.
+FetchMock.routeSequence = (pattern, responses) => {
+	let index = 0
+	FetchMock.route(pattern, () => {
+		const response = responses[Math.min(index, responses.length - 1)]
+		index++
+		return typeof response === 'function' ? response() : response
+	})
+}
+
+FetchMock.html = (body, { status = 200, headers = {}, contentType = 'text/html; charset=utf-8' } = {}) => {
+	return new Response(body, {
+		status,
+		headers: { 'content-type': contentType, ...headers },
+	})
+}
+
+// Raw bytes, for the charset tests where the encoding is the point.
+FetchMock.buffer = (buffer, { status = 200, contentType = 'text/html', headers = {} } = {}) => {
+	return new Response(buffer, {
+		status,
+		headers: { 'content-type': contentType, ...headers },
+	})
+}
+
+FetchMock.status = (status, { headers = {}, body = '' } = {}) => {
+	return new Response(body || null, { status, headers })
+}
+
+FetchMock.redirect = (location, status = 302) => {
+	return new Response(null, { status, headers: { location } })
+}
+
+//
+// A body that never ends. Proves the size cap aborts mid-stream rather than
+// buffering until the process dies.
+//
+FetchMock.oversized = (contentType = 'text/html') => {
+	const chunk = Buffer.alloc(64 * 1024, 0x61)
+	const stream = new ReadableStream({
+		pull(controller) { controller.enqueue(new Uint8Array(chunk)) },
+	})
+	return new Response(stream, { status: 200, headers: { 'content-type': contentType } })
+}
+
+// Never settles - the AbortController is the only way out.
+FetchMock.hang = () => new Promise(() => {})
+
+FetchMock.networkError = (code = 'ECONNRESET') => () => {
+	throw Object.assign(new Error(`socket ${code}`), { code })
+}
+
+FetchMock.calls = () => callLog.slice()
+FetchMock.callsTo = (pattern) => callLog.filter(entry =>
+	pattern instanceof RegExp ? pattern.test(entry.url) : entry.url === pattern)
+FetchMock.callCount = () => callLog.length

server/workers/ScrapeQueue.js

@@ -0,0 +1,466 @@
+//
+// The scrape worker.
+//
+// Owns the whole lifecycle for one URL: cache check -> robots -> polite fetch ->
+// parse -> thumbnail -> persist -> notify. Saving a note enqueues URLs and
+// returns immediately; nothing in the request path waits on the network.
+//
+// Structural fixes over the old implementation:
+//   - Promise.allSettled semantics instead of a completion counter, so a single
+//     rejection can no longer strand a batch forever.
+//   - Every fetch goes through HostQueue, so there is no unbounded fan-out.
+//   - Jobs for the same URL coalesce into one fetch, fanned out to every
+//     attachment row that wanted it.
+//   - Pending work is recoverable after a restart from attachment.scrape_state.
+//
+
+const config = require('@config/scrape')
+const db = require('@config/database')
+const UrlSafety = require('@helpers/UrlSafety')
+const HttpFetch = require('@helpers/HttpFetch')
+const HostQueue = require('@helpers/HostQueue')
+const Robots = require('@helpers/Robots')
+const Realtime = require('@helpers/Realtime')
+const SiteScrape = require('@helpers/SiteScrape')
+const ScrapeCache = require('@models/ScrapeCache')
+
+const cheerio = require('cheerio')
+
+let ScrapeQueue = module.exports = {}
+
+// attachment.scrape_state
+const STATE = ScrapeQueue.STATE = { PENDING: 0, FETCHING: 1, DONE: 2, FAILED: 3 }
+
+let settings = { ...config }
+let queue = HostQueue.shared
+let started = false
+
+// urlKey -> { promise, jobs: [] } so concurrent requests for one URL share a fetch.
+const inFlight = new Map()
+// userId -> pending job count, for backpressure.
+const perUser = new Map()
+// userId -> debounce timer, so a 10-link note triggers one refresh not ten.
+const emitTimers = new Map()
+//
+// Every job promise currently outstanding. Needed because callers like
+// recoverPending() and scanTextForWebsites() deliberately do not await their
+// batches - without this, drain() could return while fire-and-forget work was
+// still sitting in front of its first await.
+//
+const outstanding = new Set()
+
+const EMIT_DEBOUNCE_MS = 1500
+
+const now = () => Math.round(Date.now() / 1000)
+
+//
+// Test seam. Lets a test run with zero politeness spacing and no robots lookups
+// while still exercising the real code path.
+//
+ScrapeQueue.configure = (overrides = {}) => {
+	settings = { ...config, ...overrides }
+
+	queue = new HostQueue.Queue({
+		globalConcurrency: settings.maxConcurrentFetches,
+		perHostConcurrency: settings.perHostConcurrency,
+		minSpacingMs: settings.perHostMinSpacingMs,
+		maxSpacingMs: settings.perHostMaxSpacingMs,
+	})
+
+	return settings
+}
+
+ScrapeQueue.__state = () => ({
+	inFlight: inFlight.size,
+	perUser: new Map(perUser),
+	queue: queue.stats(),
+	started,
+})
+
+ScrapeQueue.__reset = () => {
+	inFlight.clear()
+	outstanding.clear()
+	perUser.clear()
+	emitTimers.forEach(timer => clearTimeout(timer))
+	emitTimers.clear()
+	settings = { ...config }
+	queue = HostQueue.shared
+	started = false
+}
+
+ScrapeQueue.start = () => {
+	started = true
+	return ScrapeQueue
+}
+
+ScrapeQueue.stop = () => {
+	started = false
+	queue.stop()
+	emitTimers.forEach(timer => clearTimeout(timer))
+	emitTimers.clear()
+}
+
+//
+// Resolves once every queued and in-flight job has settled.
+//
+// Loops because jobs enqueue follow-up work: a page fetch spawns a thumbnail
+// fetch, and a recovery batch is registered asynchronously.
+//
+ScrapeQueue.drain = async () => {
+	for (let pass = 0; pass < 20; pass++) {
+		await Promise.allSettled([...outstanding])
+		await queue.drain()
+		await Promise.allSettled([...inFlight.values()].map(entry => entry.promise))
+
+		if (!outstanding.size && !inFlight.size && !queue.size() && !queue.inflight()) {
+			return
+		}
+	}
+}
+
+ScrapeQueue.size = () => queue.size() + inFlight.size + outstanding.size
+
+//
+// Debounced notification. The client already refetches on these events, so one
+// per user per window is enough - the old code emitted once per URL.
+//
+const notifyUser = (userId) => {
+	if (emitTimers.has(userId)) { return }
+
+	const timer = setTimeout(() => {
+		emitTimers.delete(userId)
+		Realtime.emitToUser(userId, 'update_counts')
+		Realtime.emitToUser(userId, 'update_note_attachments')
+	}, EMIT_DEBOUNCE_MS)
+
+	if (timer.unref) { timer.unref() }
+	emitTimers.set(userId, timer)
+}
+
+// Tests want the emit without waiting out the debounce.
+ScrapeQueue.__flushNotifications = () => {
+	for (const [userId, timer] of [...emitTimers.entries()]) {
+		clearTimeout(timer)
+		emitTimers.delete(userId)
+		Realtime.emitToUser(userId, 'update_counts')
+		Realtime.emitToUser(userId, 'update_note_attachments')
+	}
+}
+
+const setAttachmentState = async (attachmentId, state) => {
+	if (!attachmentId) { return }
+	await db.promise().query(
+		'UPDATE attachment SET scrape_state = ?, scrape_attempts = scrape_attempts + 1 WHERE id = ?',
+		[state, attachmentId]
+	)
+}
+
+//
+// Write a successful result onto an attachment row.
+//
+// attachment.text stays PLAINTEXT - the FULLTEXT index depends on it.
+//
+const applyResultToAttachment = async (attachmentId, fields) => {
+	if (!attachmentId) { return }
+
+	await db.promise().query(
+		`UPDATE attachment
+		 SET text = ?, url = ?, last_indexed = ?, file_location = ?, scrape_state = ?,
+			 scrape_attempts = scrape_attempts + 1
+		 WHERE id = ?`,
+		[fields.text, fields.url, now(), fields.fileLocation, STATE.DONE, attachmentId]
+	)
+}
+
+//
+// A failed scrape must not leave a sentinel string in the search index. The old
+// code stored 'No site text' / 'Request Timeout' in attachment.text, which then
+// got concatenated into the note's searchable content.
+//
+const applyFailureToAttachment = async (attachmentId, error) => {
+	if (!attachmentId) { return }
+
+	const kind = (error && error.kind) || 'unknown'
+
+	// A blocked or permanently-gone URL has no preview to show, ever. Drop the
+	// shell row rather than leaving an orphan showing the bare URL.
+	const terminal = kind === 'blocked' || kind === 'robots_disallowed' ||
+		(kind === 'http' && (error.status === 404 || error.status === 410))
+
+	if (terminal) {
+		await db.promise().query('DELETE FROM attachment WHERE id = ? AND scrape_state IN (?, ?)',
+			[attachmentId, STATE.PENDING, STATE.FETCHING])
+		return
+	}
+
+	await setAttachmentState(attachmentId, STATE.FAILED)
+}
+
+//
+// Fetch and store the preview image. Never fatal: a page with no usable
+// thumbnail is still a perfectly good link preview.
+//
+const fetchThumbnail = async (imageUrl, pageUrl) => {
+	if (!imageUrl) { return null }
+
+	// Lazy require: Attachment requires this module, so a top-level require
+	// would be a cycle.
+	const Attachment = require('@models/Attachment')
+
+	try {
+		return await queue.schedule(hostKeyFor(imageUrl), () =>
+			Attachment.downloadThumbnail(imageUrl, pageUrl))
+	} catch (error) {
+		console.log('[scrape] thumbnail failed', imageUrl, error.kind || error.message)
+		return null
+	}
+}
+
+const hostKeyFor = (url) => {
+	try { return UrlSafety.registrableHost(new URL(url).hostname) }
+	catch (error) { return '(none)' }
+}
+
+//
+// Fetch one URL and write it to the cache. Returns the cache row fields.
+//
+const performFetch = async (url) => {
+	const cached = await ScrapeCache.get(url)
+
+	// Fresh and usable - zero network I/O.
+	if (ScrapeCache.isFresh(cached) && ScrapeCache.isUsable(cached)) {
+		return { fromCache: true, fields: ScrapeCache.toAttachmentFields(cached) }
+	}
+
+	// Parked failure still inside its backoff window.
+	if (ScrapeCache.isFresh(cached) && cached.fetch_state !== 'ok') {
+		const error = new Error(cached.last_error || 'previously failed')
+		error.kind = cached.fetch_state === 'blocked' ? 'blocked' : 'http'
+		error.status = cached.status || null
+		throw error
+	}
+
+	if (settings.checkRobots !== false) {
+		const allowed = await Robots.isAllowed(url)
+		if (!allowed) {
+			const error = new Error(`robots.txt disallows ${url}`)
+			error.kind = 'robots_disallowed'
+			await ScrapeCache.recordFailure(url, error)
+			throw error
+		}
+	}
+
+	let spacingMs = null
+	if (settings.checkRobots !== false) {
+		const crawlDelay = await Robots.crawlDelayMs(url)
+		if (Robots.exceedsCrawlDelayCap(crawlDelay)) {
+			const error = new Error(`Crawl-delay of ${crawlDelay}ms exceeds cap`)
+			error.kind = 'blocked'
+			await ScrapeCache.recordFailure(url, error)
+			throw error
+		}
+		spacingMs = crawlDelay
+	}
+
+	await ScrapeCache.markPending(url)
+
+	const conditional = ScrapeCache.conditionalFor(cached)
+
+	let response = null
+	try {
+		response = await queue.schedule(hostKeyFor(url), () => HttpFetch.fetchDocument(url, {
+			timeoutMs: settings.pageTimeoutMs,
+			budgetMs: settings.totalBudgetMs,
+			maxBytes: settings.maxHtmlBytes,
+			conditional,
+		}), { spacingMs })
+	} catch (error) {
+		await ScrapeCache.recordFailure(url, error)
+		throw error
+	}
+
+	// Unchanged since last time - reuse what we already stored.
+	if (response.notModified && cached) {
+		await ScrapeCache.touchNotModified(url)
+		const refreshed = await ScrapeCache.get(url)
+		return { fromCache: true, notModified: true, fields: ScrapeCache.toAttachmentFields(refreshed) }
+	}
+
+	const $ = cheerio.load(response.html || '')
+	const parsed = SiteScrape.parse($, response.finalUrl || url)
+
+	// Reuse an existing thumbnail file rather than refetching the image.
+	let fileLocation = (cached && cached.image_url === parsed.imageUrl && cached.file_location)
+		? cached.file_location
+		: await fetchThumbnail(parsed.imageUrl, response.finalUrl || url)
+
+	await ScrapeCache.putSuccess(url, {
+		...parsed,
+		status: response.status,
+		etag: response.etag,
+		lastModified: response.lastModified,
+		contentType: response.contentType,
+		fileLocation,
+	})
+
+	return {
+		fromCache: false,
+		fields: {
+			text: parsed.searchText,
+			fileLocation,
+			// Keep the URL the user wrote. The canonical URL is recorded in the
+			// cache row; rewriting attachment.url would change link behaviour.
+			url,
+		},
+	}
+}
+
+//
+// Coalesce: one network fetch per URL no matter how many notes asked for it.
+//
+const fetchCoalesced = (url) => {
+	const key = UrlSafety.urlKey(url)
+	if (!key) { return Promise.reject(Object.assign(new Error(`unusable url ${url}`), { kind: 'blocked' })) }
+
+	const existing = inFlight.get(key)
+	if (existing) { return existing.promise }
+
+	const promise = performFetch(url).finally(() => inFlight.delete(key))
+	inFlight.set(key, { promise, url })
+
+	return promise
+}
+
+const releaseUser = (userId) => {
+	const count = (perUser.get(userId) || 0) - 1
+	if (count <= 0) { perUser.delete(userId) } else { perUser.set(userId, count) }
+}
+
+//
+// Queue one URL. Resolves with the search text (or '' on failure) and never
+// rejects, so a caller can await a whole batch without special handling.
+//
+ScrapeQueue.enqueueUrl = ({ userId, noteId, attachmentId, url }) => {
+	const job = runJob({ userId, noteId, attachmentId, url })
+
+	// Registered synchronously, so drain() sees the job even before it reaches
+	// its first await.
+	outstanding.add(job)
+	job.finally(() => outstanding.delete(job))
+
+	return job
+}
+
+const runJob = async ({ userId, noteId, attachmentId, url }) => {
+	if (!settings.enabled) { return '' }
+
+	const pending = perUser.get(userId) || 0
+	if (pending >= settings.maxPendingPerUser) {
+		// Left at scrape_state = 0, so recoverPending() picks it up later.
+		// Deferred, not dropped.
+		console.log(`[scrape] deferring ${url} - user ${userId} at pending cap`)
+		return ''
+	}
+	perUser.set(userId, pending + 1)
+
+	try {
+		await setAttachmentState(attachmentId, STATE.FETCHING)
+
+		const result = await fetchCoalesced(url)
+		await applyResultToAttachment(attachmentId, result.fields)
+
+		notifyUser(userId)
+
+		return result.fields.text || ''
+	} catch (error) {
+		await applyFailureToAttachment(attachmentId, error)
+		notifyUser(userId)
+
+		console.log(`[scrape] ${url} failed: ${error.kind || 'error'} ${error.message}`)
+		return ''
+	} finally {
+		releaseUser(userId)
+	}
+}
+
+//
+// Queue a batch. allSettled, so one bad URL cannot strand the rest - this is
+// the structural replacement for the old completion counter.
+//
+ScrapeQueue.enqueueUrls = async (jobs) => {
+	if (!jobs || !jobs.length) { return [] }
+
+	const results = await Promise.allSettled(jobs.map(job => ScrapeQueue.enqueueUrl(job)))
+
+	return results.map(result => result.status === 'fulfilled' ? result.value : '')
+}
+
+//
+// Wait for one URL already queued, with a hard cap. Used by the bookmarklet so
+// it keeps responding after the scrape without being able to hang forever.
+//
+ScrapeQueue.awaitUrl = async (promise, capMs) => {
+	let timer = null
+
+	const timeout = new Promise(resolve => {
+		timer = setTimeout(() => resolve('__scrape_timeout__'), capMs || settings.totalBudgetMs)
+		if (timer.unref) { timer.unref() }
+	})
+
+	try {
+		return await Promise.race([promise, timeout])
+	} finally {
+		if (timer) { clearTimeout(timer) }
+	}
+}
+
+//
+// Crash recovery.
+//
+// The queue is in memory, so a pm2 reload loses it - but every enqueued URL
+// already has an attachment row, which makes the work re-derivable with no
+// extra table. Rows stuck mid-fetch are reset, rows still pending are
+// re-enqueued, and anything older than a day is retired so nothing
+// recirculates forever.
+//
+ScrapeQueue.recoverPending = async () => {
+	if (!settings.enabled) { return { requeued: 0, retired: 0 } }
+
+	// Anything left at FETCHING died with the previous process.
+	await db.promise().query(
+		'UPDATE attachment SET scrape_state = ? WHERE scrape_state = ?',
+		[STATE.PENDING, STATE.FETCHING]
+	)
+
+	const cutoff = now() - (24 * 60 * 60)
+
+	const [retiredResult] = await db.promise().query(
+		`UPDATE attachment SET scrape_state = ?
+		 WHERE scrape_state = ? AND enqueued_at IS NOT NULL AND enqueued_at < ?`,
+		[STATE.FAILED, STATE.PENDING, cutoff]
+	)
+
+	const [rows] = await db.promise().query(
+		`SELECT id, user_id, note_id, url FROM attachment
+		 WHERE attachment_type = 1 AND visible = 1 AND scrape_state = ?
+		 ORDER BY enqueued_at IS NULL, enqueued_at
+		 LIMIT ?`,
+		[STATE.PENDING, settings.maxQueueLength]
+	)
+
+	if (rows.length) {
+		console.log(`[scrape] recovering ${rows.length} pending URL(s) after restart`)
+	}
+
+	// Fire and forget: boot must not wait on the network.
+	// Fire and forget, but the individual jobs register themselves in
+	// `outstanding`, so drain() can still wait for them.
+	ScrapeQueue.enqueueUrls(rows.map(row => ({
+		userId: row.user_id,
+		noteId: row.note_id,
+		attachmentId: row.id,
+		url: row.url,
+	}))).catch(error => console.log('[scrape] recovery failed', error.message))
+
+	return { requeued: rows.length, retired: retiredResult.affectedRows || 0 }
+}