Adding scrape files for testing

server/tests/fixtures/html/README.md

@@ -0,0 +1,7 @@
+# HTML fixtures
+
+Frozen, hand-trimmed snapshots used by `server/tests/helpers/SiteScrape.test.js`.
+
+These are **not** fetched live. Each file isolates one parsing situation so a
+failure points at a specific branch. Add a new file rather than growing an
+existing one when covering a new case.

server/tests/fixtures/html/apple-touch-icon.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head>
+  <title>Icons By Size</title>
+  <link rel="icon" href="https://example.com/small.png" sizes="16x16">
+  <link rel="apple-touch-icon" href="https://example.com/touch-180.png" sizes="180x180">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/extensionless-og-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Generated OG Image</title>
+  <meta property="og:image" content="https://example.com/api/og?title=Hello&amp;theme=dark">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/h1-fallback.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<html>
+<head></head>
+<body><h1>Heading Is All We Have</h1><p>No title element, no meta tags.</p></body>
+</html>

server/tests/fixtures/html/imgs-missing-src.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html>
+<head><title>Broken Images</title></head>
+<body>
+  <img>
+  <img alt="no src here">
+  <img src="">
+  <img data-src="https://cdn.example.com/lazy.jpg">
+</body>
+</html>

server/tests/fixtures/html/jsonld-article.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+<head>
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@type": "NewsArticle",
+    "headline": "JSON-LD Headline",
+    "description": "Described in structured data.",
+    "image": ["https://cdn.example.com/ld-primary.jpg", "https://cdn.example.com/ld-second.jpg"]
+  }
+  </script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/jsonld-graph.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+<head>
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@graph": [
+      { "@type": "Organization", "name": "Not The Article" },
+      { "@type": "BlogPosting", "headline": "Graph Headline", "image": { "url": "https://cdn.example.com/graph.png" } }
+    ]
+  }
+  </script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/jsonld-malformed.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+<head>
+  <title>Survived Bad JSON</title>
+  <script type="application/ld+json">
+  { "@type": "Article", "headline": "unterminated,,, }
+  </script>
+  <script type="application/ld+json"></script>
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/keyword-content.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+<head><title>Beekeeping Guide</title></head>
+<body>
+  <nav>Home About Contact Login Signup Navigation Navigation</nav>
+  <script>var tracking = "tracking tracking tracking tracking tracking";</script>
+  <style>.beehive { color: red }</style>
+  <article class="content">
+    <p>Beekeeping is rewarding. Beekeeping requires patience. A beehive needs space.</p>
+    <p>The beehive must stay dry. Honey harvest comes later. Honey harvest rewards patience.</p>
+    <p>Beekeeping beginners should buy a beehive and study honey production carefully.</p>
+    <p>Honey production depends on the beehive and on steady beekeeping habits.</p>
+  </article>
+  <footer>Footer footer footer links copyright</footer>
+</body>
+</html>

server/tests/fixtures/html/malformed-unclosed.html

@@ -0,0 +1,7 @@
+<!doctype html>
+<html>
+<head><title>Unclosed Tags
+<body>
+  <div><p>Paragraph never closed
+  <ul><li>item one<li>item two
+</html>

server/tests/fixtures/html/no-metadata.html

@@ -0,0 +1,2 @@
+<!doctype html>
+<html><head></head><body></body></html>

server/tests/fixtures/html/og-complete.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <title>Fallback Title Nobody Should See</title>
+  <meta property="og:title" content="The Real Title">
+  <meta property="og:description" content="A proper social description of the page.">
+  <meta property="og:site_name" content="Example Journal">
+  <meta property="og:image" content="https://cdn.example.com/hero.jpg">
+  <meta property="og:url" content="https://example.com/canonical-post">
+  <meta name="description" content="A lesser meta description.">
+  <link rel="canonical" href="https://example.com/canonical-post">
+</head>
+<body><article><p>Body text.</p></article></body>
+</html>

server/tests/fixtures/html/protocol-relative-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Protocol Relative</title>
+  <meta property="og:image" content="//cdn.example.com/proto.jpg">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/relative-images.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head>
+  <title>Relative Assets</title>
+  <link rel="canonical" href="https://example.com/blog/2026/post">
+  <meta property="og:image" content="../images/hero.png">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/root-relative-icon.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Icon Only</title>
+  <link rel="shortcut icon" href="/favicon.ico">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/sized-images.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head><title>Sized</title></head>
+<body>
+  <img src="https://cdn.example.com/thumb.jpg" width="80" height="60">
+  <img src="https://cdn.example.com/big.jpg" width="1200" height="800">
+  <img src="https://cdn.example.com/mid.jpg" width="400" height="300">
+</body>
+</html>

server/tests/fixtures/html/srcset.html

@@ -0,0 +1,7 @@
+<!doctype html>
+<html>
+<head><title>Srcset</title></head>
+<body>
+  <img srcset="https://cdn.example.com/small.jpg 320w, https://cdn.example.com/large.jpg 1600w, https://cdn.example.com/medium.jpg 800w">
+</body>
+</html>

server/tests/fixtures/html/svg-only-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>SVG Only</title>
+  <meta property="og:image" content="https://cdn.example.com/logo.svg">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/title-only.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<html>
+<head><title>   Just   A   Title   </title></head>
+<body><h1>An H1 Heading</h1><p>No metadata here at all.</p></body>
+</html>

server/tests/fixtures/html/tracking-pixels.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+<head><title>Pixels</title></head>
+<body>
+  <img src="https://analytics.example.net/1x1.gif" width="1" height="1">
+  <img src="https://cdn.example.com/spacer.gif">
+  <img src="https://cdn.example.com/real-photo.jpg" width="800" height="600">
+</body>
+</html>

server/tests/fixtures/html/twitter-only.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html>
+<head>
+  <title>Document Title</title>
+  <meta name="twitter:title" content="Twitter Card Title">
+  <meta name="twitter:description" content="Twitter card description.">
+  <meta name="twitter:image" content="https://cdn.example.com/twitter.png">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/fixtures/html/webp-og-image.html

@@ -0,0 +1,8 @@
+<!doctype html>
+<html>
+<head>
+  <title>Modern Formats</title>
+  <meta property="og:image" content="https://cdn.example.com/hero.webp">
+</head>
+<body><p>x</p></body>
+</html>

server/tests/helpers/HostQueue.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const HostQueue = require('@helpers/HostQueue')
+
+//
+// A controllable clock that also drives the queue's own timers, so advancing
+// time is instant and spacing assertions are exact rather than flaky.
+//
+const makeClock = () => {
+	let t = 1000
+	let nextId = 1
+	const timers = new Map()
+
+	return {
+		now: () => t,
+		setTimeoutFn: (fn, ms) => {
+			const id = nextId++
+			timers.set(id, { fn, dueAt: t + ms })
+			return id
+		},
+		clearTimeoutFn: (id) => { timers.delete(id) },
+		advance: (ms) => {
+			t += ms
+			for (const [id, timer] of [...timers.entries()]) {
+				if (timer.dueAt <= t) {
+					timers.delete(id)
+					timer.fn()
+				}
+			}
+		},
+	}
+}
+
+const wireClock = (clock) => ({
+	now: clock.now,
+	setTimeoutFn: clock.setTimeoutFn,
+	clearTimeoutFn: clock.clearTimeoutFn,
+})
+
+const deferred = () => {
+	let resolve = null
+	let reject = null
+	const promise = new Promise((res, rej) => { resolve = res; reject = rej })
+	return { promise, resolve, reject }
+}
+
+describe('global concurrency', () => {
+
+	test('never exceeds the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = Array.from({ length: 6 }, (unused, i) => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(`host${i}.com`, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+				return i
+			})
+		})
+
+		// Let the first batch start.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(2)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+
+		expect(peak).toBe(2)
+	})
+
+	test('runs different hosts in parallel up to the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = ['a.com', 'b.com', 'c.com'].map(host => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(host, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+			})
+		})
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(3)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+	})
+})
+
+describe('per-host serialisation', () => {
+
+	test('runs only one request per host at a time', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+
+		const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => {
+			active++
+			peak = Math.max(peak, active)
+			await new Promise(resolve => setImmediate(resolve))
+			active--
+		}))
+
+		await Promise.all(jobs)
+
+		expect(peak).toBe(1)
+	})
+
+	test('preserves FIFO order within a host', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+		const order = []
+
+		await Promise.all([1, 2, 3, 4].map(n =>
+			queue.schedule('same.com', async () => { order.push(n) })
+		))
+
+		expect(order).toEqual([1, 2, 3, 4])
+	})
+})
+
+describe('politeness spacing', () => {
+
+	test('leaves a gap between two requests to the same host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 1000,
+			maxSpacingMs: 1000,
+			...wireClock(clock),
+		})
+
+		const startedAt = []
+
+		const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+		await first
+
+		const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+
+		// The second job must not have run yet - the host is spaced out.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(startedAt).toHaveLength(1)
+
+		clock.advance(1000)
+		await second
+
+		expect(startedAt).toHaveLength(2)
+		expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000)
+	})
+
+	test('spacing on one host does not delay a different host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 5000,
+			maxSpacingMs: 5000,
+			...wireClock(clock),
+		})
+
+		await queue.schedule('slow.com', async () => {})
+
+		// slow.com is now parked for 5s. other.com must be unaffected.
+		const ran = []
+		await queue.schedule('other.com', async () => { ran.push('other') })
+
+		expect(ran).toEqual(['other'])
+	})
+
+	test('honours an explicit spacing override (robots Crawl-delay)', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) })
+
+		await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 })
+
+		const ran = []
+		const second = queue.schedule('slow.com', async () => { ran.push(1) })
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(ran).toEqual([])
+
+		clock.advance(3000)
+		await second
+		expect(ran).toEqual([1])
+	})
+})
+
+describe('error handling', () => {
+
+	test('a rejecting task does not wedge the queue', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', async () => { throw new Error('boom') }))
+			.rejects.toThrow('boom')
+
+		await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine')
+	})
+
+	test('a task that throws synchronously is caught', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', () => { throw new Error('sync boom') }))
+			.rejects.toThrow('sync boom')
+
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('one failing host does not block others', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const results = await Promise.allSettled([
+			queue.schedule('bad.com', async () => { throw new Error('nope') }),
+			queue.schedule('good.com', async () => 'ok'),
+		])
+
+		expect(results[0].status).toBe('rejected')
+		expect(results[1].value).toBe('ok')
+	})
+})
+
+describe('drain and stop', () => {
+
+	test('drain resolves when everything has finished', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+		const done = []
+
+		;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {}))
+
+		await queue.drain()
+
+		expect(done).toHaveLength(3)
+		expect(queue.size()).toBe(0)
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('drain resolves immediately on an idle queue', async () => {
+		const queue = new HostQueue.Queue()
+		await expect(queue.drain()).resolves.toBeUndefined()
+	})
+
+	test('stop rejects queued work and refuses new work', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const queued = queue.schedule('b.com', async () => 'never')
+
+		await new Promise(resolve => setImmediate(resolve))
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+		await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped')
+
+		gate.resolve('done')
+		await expect(first).resolves.toBe('done')
+	})
+})
+
+describe('event loop', () => {
+
+	//
+	// Regression: the politeness timer used to be unref'd, so if nothing else
+	// held the event loop open Node exited and queued jobs were abandoned with
+	// their promises never settling.
+	//
+	test('a spaced job still resolves with nothing else holding the loop open', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 })
+
+		await queue.schedule('same.com', async () => 'first')
+
+		// Only the internal timer can resume this one.
+		await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second')
+	})
+
+	test('stop() clears the pending timer so shutdown is not blocked', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 })
+
+		await queue.schedule('same.com', async () => 'first')
+		const queued = queue.schedule('same.com', async () => 'never')
+
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+	})
+})
+
+describe('stats', () => {
+
+	test('reports pending, running and host counts', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const second = queue.schedule('b.com', async () => {})
+
+		await new Promise(resolve => setImmediate(resolve))
+
+		const stats = queue.stats()
+		expect(stats.running).toBe(1)
+		expect(stats.pending).toBe(1)
+
+		gate.resolve()
+		await Promise.all([first, second])
+	})
+})

server/tests/helpers/HttpFetch.test.js

@@ -0,0 +1,479 @@
+require('module-alias/register')
+
+const HttpFetch = require('@helpers/HttpFetch')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+beforeEach(() => {
+	FetchMock.install()
+	// Every hostname in these tests resolves to one public address, so the
+	// safety gate passes and we are testing HttpFetch rather than DNS.
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+	// No real wall-time in backoff.
+	HttpFetch.__setSleep(() => Promise.resolve())
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	HttpFetch.__resetSleep()
+})
+
+describe('fetchDocument happy path', () => {
+
+	test('returns decoded html, status and final url', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<title>Hello</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.status).toBe(200)
+		expect(result.html).toContain('<title>Hello</title>')
+		expect(result.finalUrl).toBe('https://example.com/')
+		expect(result.notModified).toBe(false)
+	})
+
+	test('surfaces etag and last-modified for later conditional requests', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>', {
+			headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.etag).toBe('W/"abc"')
+		expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+})
+
+describe('fetchDocument browser identity', () => {
+
+	test('sends a complete, coherent header set and never claims to be Googlebot', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/')
+		const headers = FetchMock.calls()[0].headers
+
+		expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/)
+		expect(headers['User-Agent']).not.toMatch(/Googlebot/i)
+		expect(headers['Accept']).toContain('text/html')
+		expect(headers['Accept-Language']).toBeTruthy()
+		expect(headers['Accept-Encoding']).toContain('gzip')
+	})
+
+	test('gives the same host the same profile across calls', async () => {
+		FetchMock.route(/example\.com/, () => FetchMock.html('<p>x</p>'))
+
+		const first = await HttpFetch.fetchDocument('https://example.com/a')
+		const second = await HttpFetch.fetchDocument('https://example.com/b')
+
+		expect(first.profileId).toBe(second.profileId)
+	})
+
+	test('pairs sec-ch-ua only with Chromium profiles', () => {
+		const chrome = BrowserProfiles.profileById('chrome-win')
+		const firefox = BrowserProfiles.profileById('firefox-win')
+		const safari = BrowserProfiles.profileById('safari-mac')
+
+		const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' })
+		const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' })
+		const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' })
+
+		expect(chromeHeaders['sec-ch-ua']).toBeTruthy()
+		expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Firefox sends Sec-Fetch-* but no client hints.
+		expect(firefoxHeaders['sec-ch-ua']).toBeUndefined()
+		expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Safari sends neither.
+		expect(safariHeaders['sec-ch-ua']).toBeUndefined()
+		expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined()
+	})
+
+	test('never pairs a chrome UA with a firefox accept header', () => {
+		BrowserProfiles.PROFILES.forEach(profile => {
+			if (profile.family === 'chrome') {
+				expect(profile.ua).toMatch(/Chrome\//)
+				expect(profile.accept).toContain('application/signed-exchange')
+			}
+			if (profile.family === 'firefox') {
+				expect(profile.ua).toMatch(/Firefox\//)
+				expect(profile.accept).not.toContain('signed-exchange')
+			}
+		})
+	})
+})
+
+describe('fetchDocument redirects', () => {
+
+	test('follows a single hop and reports the final url', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301))
+		FetchMock.route('https://example.com/b', () => FetchMock.html('<title>B</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+
+		expect(result.finalUrl).toBe('https://example.com/b')
+		expect(result.html).toContain('B')
+	})
+
+	test('resolves a relative Location header', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('/c', 302))
+		FetchMock.route('https://example.com/c', () => FetchMock.html('<title>C</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+		expect(result.finalUrl).toBe('https://example.com/c')
+	})
+
+	test('detects a redirect loop', async () => {
+		FetchMock.route('https://example.com/a', () => FetchMock.redirect('https://example.com/b', 302))
+		FetchMock.route('https://example.com/b', () => FetchMock.redirect('https://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'redirect_loop' })
+	})
+
+	test('gives up past the redirect cap', async () => {
+		let n = 0
+		FetchMock.route(/https:\/\/example\.com\/hop/, () => FetchMock.redirect(`https://example.com/hop${++n}`, 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/hop'))
+			.rejects.toMatchObject({ kind: 'too_many_redirects' })
+	})
+
+	// The SSRF case that matters: the first URL is perfectly public.
+	test('blocks a redirect pointing at loopback', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://127.0.0.1:3000/admin', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('blocks a redirect pointing at the cloud metadata endpoint', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://169.254.169.254/latest/meta-data/', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('refuses an https to http downgrade', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+})
+
+describe('fetchDocument retries', () => {
+
+	test('retries a 500 and succeeds', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(500),
+			FetchMock.html('<title>Recovered</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('Recovered')
+		expect(FetchMock.callCount()).toBe(2)
+	})
+
+	test('gives up after exhausting retries', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(503))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 503 })
+
+		// 1 initial attempt + config.retries
+		expect(FetchMock.callCount()).toBe(3)
+	})
+
+	test('does NOT retry a 404 - it is an answer, not a failure', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('does NOT retry a 403', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(403))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/')).rejects.toMatchObject({ status: 403 })
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('honours a short Retry-After on a 429', async () => {
+		const waits = []
+		HttpFetch.__setSleep((ms) => { waits.push(ms); return Promise.resolve() })
+
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(429, { headers: { 'retry-after': '2' } }),
+			FetchMock.html('<title>OK</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('OK')
+		expect(waits[0]).toBe(2000)
+	})
+
+	test('refuses to sit on an hour-long Retry-After', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(429, { headers: { 'retry-after': '3600' } }))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 429 })
+
+		// Surfaced for the cache to park the URL rather than retried.
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('retries a network error', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.networkError('ECONNRESET'),
+			FetchMock.html('<title>Up</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('Up')
+	})
+})
+
+describe('fetchDocument limits and gates', () => {
+
+	test('times out on a server that never responds', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.hang())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { timeoutMs: 60, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'timeout' })
+	})
+
+	test('aborts a body that exceeds the cap mid-stream', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.oversized())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { maxBytes: 128 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects an oversized declared content-length without reading the body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>small</p>', {
+			headers: { 'content-length': String(50 * 1024 * 1024) },
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('refuses a non-html content type before parsing', async () => {
+		FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'application/pdf',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('refuses a binary masquerading behind an image content type', async () => {
+		FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'image/png',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+})
+
+describe('fetchDocument charset handling', () => {
+
+	test('decodes windows-1252 declared in the content-type header', async () => {
+		// 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8.
+		const body = Buffer.concat([
+			Buffer.from('<title>', 'latin1'),
+			Buffer.from([0x93]),
+			Buffer.from('caf', 'latin1'),
+			Buffer.from([0xe9, 0x94]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=windows-1252',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('“café”')
+		expect(result.charset).toBe('windows-1252')
+	})
+
+	test('falls back to a meta charset when the header says nothing', async () => {
+		const body = Buffer.concat([
+			Buffer.from('<meta charset="windows-1252"><title>caf', 'latin1'),
+			Buffer.from([0xe9]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, { contentType: 'text/html' }))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('header charset wins over the meta tag', async () => {
+		const body = Buffer.from('<meta charset="windows-1252"><title>café</title>', 'utf8')
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=utf-8',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('survives a bogus charset label', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(Buffer.from('<p>ok</p>'), {
+			contentType: 'text/html; charset=not-a-real-charset',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('ok')
+	})
+})
+
+describe('fetchDocument conditional requests', () => {
+
+	test('sends If-None-Match and If-Modified-Since when given', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		})
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['If-None-Match']).toBe('W/"v1"')
+		expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+
+	test('reports a 304 as notModified with no body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(304))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"' },
+		})
+
+		expect(result.notModified).toBe(true)
+		expect(result.status).toBe(304)
+		expect(result.html).toBeNull()
+	})
+})
+
+describe('fetchBinary', () => {
+
+	const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+
+	test('returns the buffer for a real image', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		const result = await HttpFetch.fetchBinary('https://example.com/i.png')
+
+		expect(result.mime).toBe('image/png')
+		expect(result.buffer.equals(pngMagic)).toBe(true)
+	})
+
+	// The old code wrote a 404 HTML error page to disk and handed it to gm.
+	test('refuses an html error page served where an image was expected', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.html('<h1>404</h1>'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('propagates a 404 status rather than writing anything', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+	})
+
+	test('enforces the image size cap', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.oversized('image/png'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { maxBytes: 64 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects svg by default', async () => {
+		FetchMock.route('https://example.com/i.svg', () => FetchMock.buffer(Buffer.from('<svg/>'), {
+			contentType: 'image/svg+xml',
+		}))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.svg', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('sends image-appropriate fetch metadata and a page referer', async () => {
+		FetchMock.route('https://cdn.example.net/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		await HttpFetch.fetchBinary('https://cdn.example.net/i.png', { pageUrl: 'https://example.com/post' })
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['Referer']).toBe('https://example.com/post')
+		expect(headers['Accept']).toContain('image/')
+		if (headers['Sec-Fetch-Dest']) {
+			expect(headers['Sec-Fetch-Dest']).toBe('image')
+			expect(headers['Sec-Fetch-Site']).toBe('cross-site')
+		}
+	})
+})
+
+describe('transient vs terminal safety failures', () => {
+
+	// A private address is a permanent refusal; a DNS failure means the host
+	// may just be down. Classifying both as "blocked" parked transient
+	// failures for the full cache TTL.
+	test('classifies a private address as blocked', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' })
+	})
+
+	test('classifies a DNS failure as network, not blocked', async () => {
+		dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' })
+	})
+
+	test('retries a DNS failure, since it may be transient', async () => {
+		let attempts = 0
+		dnsSpy.mockImplementation(async () => {
+			attempts++
+			if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) }
+			return [{ address: '93.184.216.34', family: 4 }]
+		})
+		FetchMock.route('https://flaky.example/', () => FetchMock.html('<title>Back up</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://flaky.example/')
+		expect(result.html).toContain('Back up')
+		expect(attempts).toBe(2)
+	})
+
+	test('does NOT retry a private address', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' })
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})
+
+describe('safety gate on the initial url', () => {
+
+	test('refuses to open a connection to a private address at all', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>should never be reached</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked' })
+
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})

server/tests/helpers/Robots.test.js

@@ -0,0 +1,173 @@
+require('module-alias/register')
+
+const Robots = require('@helpers/Robots')
+const config = require('@config/scrape')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+const UA = config.robotsUaToken
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	Robots.__resetCache()
+})
+
+const serveRobots = (body, origin = 'https://example.com') => {
+	FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
+		status: 200,
+		headers: { 'content-type': 'text/plain' },
+	}))
+}
+
+describe('Robots.parse', () => {
+
+	test('groups consecutive user-agent lines into one rule block', () => {
+		const rules = Robots.parse([
+			'User-agent: alpha',
+			'User-agent: beta',
+			'Disallow: /private',
+		].join('\n'))
+
+		expect(rules.groups).toHaveLength(1)
+		expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
+	})
+
+	test('ignores comments and blank lines', () => {
+		const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
+	})
+
+	test('treats an empty Disallow as no restriction', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow:')
+		expect(rules.groups[0].rules).toHaveLength(0)
+	})
+
+	test('parses crawl-delay', () => {
+		const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
+		expect(rules.groups[0].crawlDelayMs).toBe(2500)
+	})
+
+	test('an empty file allows everything', () => {
+		expect(Robots.parse('').allowAll).toBe(true)
+		expect(Robots.parse(null).allowAll).toBe(true)
+	})
+})
+
+describe('Robots.isPathAllowed', () => {
+
+	test('respects a simple disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /private')
+		expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
+	})
+
+	test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
+		expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
+	})
+
+	test('Allow beats Disallow at equal specificity', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
+		expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
+	})
+
+	test('supports the * wildcard', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
+		expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
+	})
+
+	test('supports the $ anchor', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /x$')
+		expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
+	})
+
+	test('a named group beats the wildcard group even when listed later', () => {
+		const rules = Robots.parse([
+			'User-agent: *',
+			'Disallow: /',
+			'',
+			`User-agent: ${UA}`,
+			'Disallow:',
+		].join('\n'))
+
+		expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
+		expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
+	})
+})
+
+describe('Robots.isAllowed over the network', () => {
+
+	test('refuses a disallowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
+	})
+
+	test('permits an allowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
+	})
+
+	// Fail-open: a missing robots.txt is not a refusal.
+	test('allows everything when robots.txt 404s', async () => {
+		FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('allows everything when robots.txt cannot be reached', async () => {
+		FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('caches per origin - two queries, one network call', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+
+		await Robots.isAllowed('https://example.com/a')
+		await Robots.isAllowed('https://example.com/secret/b')
+
+		expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
+	})
+
+	test('keeps origins separate', async () => {
+		serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
+		serveRobots('User-agent: *\nDisallow:', 'https://open.com')
+
+		await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
+		await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
+	})
+
+	test('returns false for an unparseable url rather than throwing', async () => {
+		await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
+	})
+})
+
+describe('Robots.crawlDelayMs', () => {
+
+	test('reports the declared delay', async () => {
+		serveRobots('User-agent: *\nCrawl-delay: 5')
+		await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
+	})
+
+	test('is null when none is declared', async () => {
+		serveRobots('User-agent: *\nDisallow: /x')
+		await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
+	})
+
+	test('flags a delay beyond the cap instead of parking the job forever', async () => {
+		expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
+		expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
+		expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
+	})
+})

server/tests/helpers/SiteScrape.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const fs = require('fs')
+const path = require('path')
+const cheerio = require('cheerio')
+
+const SiteScrape = require('@helpers/SiteScrape')
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')
+
+// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
+const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))
+
+const PAGE_URL = 'https://example.com/blog/2026/post'
+
+describe('getTitle', () => {
+
+	test('prefers og:title over the title element', () => {
+		expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
+	})
+
+	test('falls back to twitter:title', () => {
+		expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
+	})
+
+	test('falls back to JSON-LD headline', () => {
+		expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
+	})
+
+	test('falls back to the title element and collapses whitespace', () => {
+		expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
+	})
+
+	test('falls back to an h1 when there is no title element', () => {
+		expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
+	})
+
+	test('returns empty string for a page with nothing', () => {
+		expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
+	})
+})
+
+describe('getDescription', () => {
+
+	test('prefers og:description', () => {
+		expect(SiteScrape.getDescription(load('og-complete.html')))
+			.toBe('A proper social description of the page.')
+	})
+
+	test('falls back to twitter:description', () => {
+		expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
+	})
+
+	test('falls back to JSON-LD description', () => {
+		expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
+	})
+
+	test('is empty when no description exists', () => {
+		expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
+	})
+})
+
+describe('getJsonLd', () => {
+
+	test('reads a plain block', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
+	})
+
+	test('digs the content node out of an @graph', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
+	})
+
+	// Malformed JSON-LD is extremely common and must never throw.
+	test('survives malformed JSON without throwing', () => {
+		expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
+		expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
+	})
+
+	test('is null when absent', () => {
+		expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
+	})
+})
+
+describe('getCanonicalUrl', () => {
+
+	test('uses link rel=canonical', () => {
+		expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
+			.toBe('https://example.com/canonical-post')
+	})
+
+	test('falls back to the requested url', () => {
+		expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
+	})
+})
+
+describe('getDisplayImage', () => {
+
+	test('prefers og:image', () => {
+		expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.jpg')
+	})
+
+	test('falls back to twitter:image', () => {
+		expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/twitter.png')
+	})
+
+	test('falls back to the JSON-LD image, taking the first of an array', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/ld-primary.jpg')
+	})
+
+	test('reads an object-form JSON-LD image', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/graph.png')
+	})
+
+	// Resolution against the canonical base, not string concatenation.
+	test('resolves a relative path against the canonical url', () => {
+		expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
+			.toBe('https://example.com/blog/images/hero.png')
+	})
+
+	test('resolves a root-relative icon', () => {
+		expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
+			.toBe('https://example.com/favicon.ico')
+	})
+
+	test('resolves a protocol-relative url', () => {
+		expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/proto.jpg')
+	})
+
+	// The old substring filter dropped all of these.
+	test('accepts .webp', () => {
+		expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.webp')
+	})
+
+	test('accepts an extensionless og:image endpoint', () => {
+		expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
+			.toBe('https://example.com/api/og?title=Hello&theme=dark')
+	})
+
+	test('rejects svg while allowSvgThumbnails is off', () => {
+		expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
+	})
+
+	test('picks the largest srcset candidate', () => {
+		expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/large.jpg')
+	})
+
+	test('prefers the largest inline image by declared area', () => {
+		expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/big.jpg')
+	})
+
+	test('skips tracking pixels and spacers', () => {
+		expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/real-photo.jpg')
+	})
+
+	// Regression: the old code read element.attribs.src directly and threw.
+	test('does not throw on img tags with no src, and finds a data-src', () => {
+		const $ = load('imgs-missing-src.html')
+		expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
+		expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
+	})
+
+	test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
+		expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
+			.toBe('https://example.com/touch-180.png')
+	})
+
+	test('returns empty string when there is no usable image', () => {
+		expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
+	})
+
+	test('survives malformed markup', () => {
+		expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
+	})
+})
+
+describe('getImageCandidates', () => {
+
+	test('returns an ordered, deduplicated list', () => {
+		const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
+		expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
+	})
+
+	test('is empty rather than null for a bare page', () => {
+		expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
+	})
+})
+
+describe('getKeywords', () => {
+
+	test('extracts page terms and ignores nav, script and footer chrome', () => {
+		const keywords = SiteScrape.getKeywords(load('keyword-content.html'))
+
+		expect(keywords).toMatch(/^Keywords: /)
+		expect(keywords).toMatch(/beekeeping|beehive|honey/)
+		expect(keywords).not.toMatch(/navigation/)
+		expect(keywords).not.toMatch(/tracking/)
+		expect(keywords).not.toMatch(/footer/)
+	})
+
+	test('returns empty string when there is nothing to count', () => {
+		expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
+	})
+
+	test('does not log page content', () => {
+		// The old implementation dumped up to 6000 chars of every page.
+		const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
+		SiteScrape.getKeywords(load('keyword-content.html'))
+		expect(spy).not.toHaveBeenCalled()
+		spy.mockRestore()
+	})
+})
+
+describe('getHostName', () => {
+
+	test('returns the https origin', () => {
+		expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
+	})
+
+	test('returns empty string for a bad url instead of throwing', () => {
+		expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
+		expect(SiteScrape.getHostName('nonsense')).toBe('')
+	})
+})
+
+describe('buildSearchText', () => {
+
+	test('joins the parts that exist', () => {
+		const text = SiteScrape.buildSearchText({
+			title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
+		})
+		expect(text).toBe('T\nD\nS\nKeywords: a, b')
+	})
+
+	test('skips empty parts', () => {
+		expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
+	})
+
+	test('caps the length so it cannot bloat attachment.text', () => {
+		const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
+		expect(text.length).toBeLessThanOrEqual(8000)
+	})
+
+	// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
+	test('never produces the old failure sentinels', () => {
+		const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
+		expect(text).toBe('')
+		expect(text).not.toContain('No site text')
+		expect(text).not.toContain('Request Timeout')
+	})
+})
+
+describe('parse', () => {
+
+	test('returns everything a link preview needs in one call', () => {
+		const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)
+
+		expect(result).toMatchObject({
+			title: 'The Real Title',
+			description: 'A proper social description of the page.',
+			siteName: 'Example Journal',
+			canonicalUrl: 'https://example.com/canonical-post',
+			imageUrl: 'https://cdn.example.com/hero.jpg',
+			hostname: 'https://example.com',
+		})
+		expect(result.searchText).toContain('The Real Title')
+		expect(result.searchText).toContain('A proper social description')
+	})
+
+	test('degrades to empty fields rather than throwing on a bare page', () => {
+		const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)
+
+		expect(result.title).toBe('')
+		expect(result.imageUrl).toBe('')
+		expect(result.searchText).toBe('example.com')
+	})
+})
+
+//
+// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
+// note text and what string is stored in attachment.url. These tests pin the
+// CURRENT behaviour so the rewrite around it cannot drift link detection.
+//
+describe('getCleanUrls (pinned, must not change)', () => {
+
+	test('finds bare and full urls', () => {
+		expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
+			.toEqual(['https://example.com/a', 'https://www.foo.com'])
+	})
+
+	test('still forces http to https', () => {
+		expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
+	})
+
+	test('still prepends https to a bare www host', () => {
+		expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
+	})
+
+	test('deduplicates', () => {
+		expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
+	})
+
+	test('returns an empty array when there are no urls', () => {
+		expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
+	})
+
+	test('returns an empty array for empty text', () => {
+		expect(SiteScrape.getCleanUrls('')).toEqual([])
+	})
+})

server/tests/helpers/UrlSafety.test.js

@@ -0,0 +1,189 @@
+require('module-alias/register')
+
+const UrlSafety = require('@helpers/UrlSafety')
+
+describe('UrlSafety.isPrivateIp', () => {
+
+	const privateAddresses = [
+		'0.0.0.0', '10.0.0.1', '10.255.255.254', '127.0.0.1', '127.1.1.1',
+		'169.254.1.1', '169.254.169.254',  // cloud metadata endpoint
+		'172.16.0.1', '172.31.255.254', '192.168.1.1', '192.0.0.1', '192.0.2.5',
+		'198.18.0.1', '100.64.0.1', '100.127.255.254', '224.0.0.1', '255.255.255.255',
+		'::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1',
+		'::ffff:127.0.0.1', '::ffff:10.0.0.1', '64:ff9b::127.0.0.1',
+	]
+
+	privateAddresses.forEach(address => {
+		test(`blocks ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(true)
+		})
+	})
+
+	const publicAddresses = ['1.1.1.1', '8.8.8.8', '93.184.216.34', '172.32.0.1', '2606:4700::1111']
+
+	publicAddresses.forEach(address => {
+		test(`allows ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(false)
+		})
+	})
+
+	test('treats a non-IP string as unsafe', () => {
+		expect(UrlSafety.isPrivateIp('not-an-ip')).toBe(true)
+		expect(UrlSafety.isPrivateIp('')).toBe(true)
+	})
+})
+
+describe('UrlSafety.parse', () => {
+
+	test('rejects non-http schemes', () => {
+		expect(UrlSafety.parse('file:///etc/passwd')).toBeNull()
+		expect(UrlSafety.parse('ftp://example.com/x')).toBeNull()
+		expect(UrlSafety.parse('javascript:alert(1)')).toBeNull()
+		expect(UrlSafety.parse('data:text/html,<h1>x</h1>')).toBeNull()
+	})
+
+	test('rejects garbage and empty input', () => {
+		expect(UrlSafety.parse('')).toBeNull()
+		expect(UrlSafety.parse(null)).toBeNull()
+		expect(UrlSafety.parse('not a url at all')).toBeNull()
+	})
+
+	test('rejects URLs longer than the attachment.url column', () => {
+		const long = 'https://example.com/' + 'a'.repeat(1200)
+		expect(UrlSafety.parse(long)).toBeNull()
+	})
+
+	test('accepts ordinary http and https', () => {
+		expect(UrlSafety.parse('https://example.com/a').href).toBe('https://example.com/a')
+		expect(UrlSafety.parse('http://example.com/a').href).toBe('http://example.com/a')
+	})
+})
+
+describe('UrlSafety.normalizeForKey / urlKey', () => {
+
+	test('strips tracking params so the cache key is stable', () => {
+		const bare = UrlSafety.urlKey('https://example.com/post')
+		const noisy = UrlSafety.urlKey('https://example.com/post?utm_source=x&utm_medium=y&fbclid=z')
+		expect(noisy).toBe(bare)
+	})
+
+	test('keeps meaningful query params', () => {
+		const a = UrlSafety.urlKey('https://example.com/search?q=cats')
+		const b = UrlSafety.urlKey('https://example.com/search?q=dogs')
+		expect(a).not.toBe(b)
+	})
+
+	test('is insensitive to param order, host case, fragment and trailing host dot', () => {
+		const canonical = UrlSafety.urlKey('https://example.com/p?a=1&b=2')
+		expect(UrlSafety.urlKey('https://example.com/p?b=2&a=1')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://EXAMPLE.com/p?a=1&b=2')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com/p?a=1&b=2#section')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com./p?a=1&b=2')).toBe(canonical)
+	})
+
+	test('distinguishes http from https', () => {
+		expect(UrlSafety.urlKey('http://example.com/')).not.toBe(UrlSafety.urlKey('https://example.com/'))
+	})
+
+	test('returns a 64-char hex digest', () => {
+		expect(UrlSafety.urlKey('https://example.com/')).toMatch(/^[0-9a-f]{64}$/)
+	})
+
+	test('returns null for an unusable URL', () => {
+		expect(UrlSafety.urlKey('javascript:alert(1)')).toBeNull()
+	})
+})
+
+describe('UrlSafety.sameResource', () => {
+
+	// This is what stops scanTextForWebsites deleting and re-scraping a row
+	// every time a note is saved.
+	test('treats http/https, www. and trailing slash as the same resource', () => {
+		expect(UrlSafety.sameResource('http://foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://www.foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com/a/', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com', 'https://foo.com/')).toBe(true)
+	})
+
+	test('does not conflate different paths, hosts or queries', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://foo.com/b')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://bar.com/a')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a?x=1', 'https://foo.com/a?x=2')).toBe(false)
+	})
+
+	test('is false when either side is unusable', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'javascript:x')).toBe(false)
+	})
+})
+
+describe('UrlSafety.assertFetchable', () => {
+
+	const expectRejection = async (url, reason) => {
+		await expect(UrlSafety.assertFetchable(url)).rejects.toMatchObject({ name: 'SafetyError', reason })
+	}
+
+	test('refuses non-http schemes', async () => {
+		await expectRejection('file:///etc/passwd', 'bad_url')
+	})
+
+	test('refuses embedded credentials', async () => {
+		await expectRejection('https://user:[email protected]/', 'credentials_in_url')
+	})
+
+	test('refuses localhost and internal suffixes', async () => {
+		await expectRejection('http://localhost:3000/', 'blocked_host')
+		await expectRejection('http://db.internal/', 'blocked_host')
+		await expectRejection('http://thing.local/', 'blocked_host')
+		await expectRejection('http://box.lan/', 'blocked_host')
+	})
+
+	test('refuses literal private addresses without touching DNS', async () => {
+		await expectRejection('http://127.0.0.1:3000/', 'private_ip')
+		await expectRejection('http://169.254.169.254/latest/meta-data/', 'private_ip')
+		await expectRejection('http://10.1.2.3/', 'private_ip')
+		await expectRejection('http://[::1]:8080/', 'private_ip')
+	})
+
+	test('allows a literal public address', async () => {
+		await expect(UrlSafety.assertFetchable('http://1.1.1.1/')).resolves.toEqual(['1.1.1.1'])
+	})
+
+	test('refuses a hostname that resolves to a private address', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '192.168.0.5', family: 4 }])
+
+		await expectRejection('https://rebind.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('refuses when ANY address in a round-robin record is private', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([
+			{ address: '93.184.216.34', family: 4 },
+			{ address: '127.0.0.1', family: 4 },
+		])
+
+		await expectRejection('https://mixed.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('allows a hostname that resolves entirely to public addresses', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+
+		await expect(UrlSafety.assertFetchable('https://example.com/')).resolves.toEqual(['93.184.216.34'])
+
+		spy.mockRestore()
+	})
+
+	test('surfaces a DNS failure as dns_failed', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+
+		await expectRejection('https://nonexistent.example/', 'dns_failed')
+
+		spy.mockRestore()
+	})
+})

server/tests/models/ScrapeQueue.test.js

@@ -0,0 +1,354 @@
+require('module-alias/register')
+
+const db = require('@config/database')
+const User = require('@models/User')
+const Attachment = require('@models/Attachment')
+const ScrapeCache = require('@models/ScrapeCache')
+const ScrapeQueue = require('@workers/ScrapeQueue')
+const Schema = require('@helpers/Schema')
+const Robots = require('@helpers/Robots')
+const HttpFetch = require('@helpers/HttpFetch')
+const UrlSafety = require('@helpers/UrlSafety')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+
+const testUserName = 'jestTestUserScrapeQueue'
+const password = 'Beans19934!!!'
+
+let userId = null
+let dnsSpy = null
+
+const page = (title, description) => FetchMock.html(`
+	<html><head>
+		<title>${title}</title>
+		<meta property="og:description" content="${description || 'a description'}">
+	</head><body><p>body</p></body></html>
+`)
+
+// Clear cache rows for a URL so each test starts from a real miss.
+const clearCache = (url) => db.promise()
+	.query('DELETE FROM scrape_cache WHERE url_hash = ?', [UrlSafety.urlKey(url)])
+
+const attachmentRow = (id) => db.promise()
+	.query('SELECT * FROM attachment WHERE id = ?', [id])
+	.then(([rows]) => rows[0] || null)
+
+beforeAll(async () => {
+	await Schema.ensureScrapeSchema()
+
+	try {
+		const existing = await User.getByUserName(testUserName)
+		if (existing && existing.id) { await User.deleteUser(existing.id, password) }
+	} catch (error) { /* first run */ }
+
+	const registered = await User.register(testUserName, password)
+	userId = registered.userId
+})
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+	HttpFetch.__setSleep(() => Promise.resolve())
+
+	// Exercise the real code path with politeness spacing and robots disabled.
+	ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	HttpFetch.__resetSleep()
+	ScrapeQueue.__reset()
+})
+
+afterAll(async () => {
+	await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
+	db.end()
+})
+
+describe('fetch and persist', () => {
+
+	test('writes parsed metadata onto the attachment row as plaintext', async () => {
+		const url = 'https://example.com/post-one'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Post One', 'the first post'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+
+		expect(row.text).toContain('Post One')
+		expect(row.text).toContain('the first post')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+		// Must stay unencrypted - the FULLTEXT index depends on it.
+		expect(row.text).not.toMatch(/^[0-9a-f]{64}/)
+	})
+
+	test('keeps the url the user wrote rather than the canonical url', async () => {
+		const url = 'https://example.com/original'
+		await clearCache(url)
+
+		FetchMock.route(url, () => FetchMock.html(`
+			<html><head><title>Canonical Test</title>
+			<link rel="canonical" href="https://example.com/different-canonical">
+			</head><body>x</body></html>
+		`))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+		expect(row.url).toBe(url)
+
+		// The canonical URL is recorded in the cache, not on the attachment.
+		const cached = await ScrapeCache.get(url)
+		expect(cached.canonical_url).toBe('https://example.com/different-canonical')
+	})
+})
+
+describe('caching', () => {
+
+	test('a second request for the same url makes zero network calls', async () => {
+		const url = 'https://example.com/cached'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Cached Page'))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		const callsAfterFirst = FetchMock.callCount()
+		expect(callsAfterFirst).toBeGreaterThan(0)
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		expect(FetchMock.callCount()).toBe(callsAfterFirst)
+
+		// The cached row still populates the second attachment.
+		const row = await attachmentRow(secondId)
+		expect(row.text).toContain('Cached Page')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('two notes sharing a url coalesce into one fetch', async () => {
+		const url = 'https://example.com/shared'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Shared Page'))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+
+		// Enqueued together, so they overlap in flight.
+		await ScrapeQueue.enqueueUrls([
+			{ userId, noteId: null, attachmentId: firstId, url },
+			{ userId, noteId: null, attachmentId: secondId, url },
+		])
+
+		expect(FetchMock.callsTo(url)).toHaveLength(1)
+
+		const rows = await Promise.all([attachmentRow(firstId), attachmentRow(secondId)])
+		rows.forEach(row => {
+			expect(row.text).toContain('Shared Page')
+			expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+		})
+	})
+
+	test('a stale entry revalidates and a 304 reuses the stored text', async () => {
+		const url = 'https://example.com/revalidate'
+		await clearCache(url)
+
+		FetchMock.routeSequence(url, [
+			FetchMock.html('<html><head><title>Original Title</title></head><body>x</body></html>', {
+				headers: { etag: 'W/"v1"' },
+			}),
+			FetchMock.status(304),
+		])
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		// Force staleness so the next pass revalidates instead of hitting cache.
+		await db.promise().query('UPDATE scrape_cache SET expires_at = ? WHERE url_hash = ?',
+			[Math.round(Date.now() / 1000) - 10, UrlSafety.urlKey(url)])
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		// Conditional request was sent.
+		const conditional = FetchMock.callsTo(url).find(call => call.headers['If-None-Match'])
+		expect(conditional).toBeTruthy()
+
+		const row = await attachmentRow(secondId)
+		expect(row.text).toContain('Original Title')
+	})
+})
+
+describe('failure handling', () => {
+
+	test('a 500 leaves the row failed and never writes a sentinel string', async () => {
+		const url = 'https://example.com/broken'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(500))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		const row = await attachmentRow(attachmentId)
+
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.FAILED)
+		// The old code stored these in attachment.text and fed them to FULLTEXT.
+		expect(row.text).not.toContain('No site text')
+		expect(row.text).not.toContain('Request Timeout')
+	})
+
+	test('a 404 removes the shell row rather than orphaning it', async () => {
+		const url = 'https://example.com/gone'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(404))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+	})
+
+	test('a blocked url removes the shell row and makes no request', async () => {
+		const url = 'http://127.0.0.1:9999/internal'
+		FetchMock.route(/.*/, () => page('should never load'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('one failing url does not strand the rest of a batch', async () => {
+		const good = 'https://example.com/good-one'
+		const bad = 'https://example.com/bad-one'
+		await clearCache(good)
+		await clearCache(bad)
+
+		FetchMock.route(good, () => page('Good One'))
+		FetchMock.route(bad, () => FetchMock.status(500))
+
+		const goodId = await Attachment.createPendingUrlAttachment(userId, null, good)
+		const badId = await Attachment.createPendingUrlAttachment(userId, null, bad)
+
+		// The old counter-based implementation never settled in this case.
+		const texts = await ScrapeQueue.enqueueUrls([
+			{ userId, noteId: null, attachmentId: badId, url: bad },
+			{ userId, noteId: null, attachmentId: goodId, url: good },
+		])
+
+		expect(texts).toHaveLength(2)
+		expect((await attachmentRow(goodId)).text).toContain('Good One')
+	})
+
+	test('a failed url is not refetched while inside its backoff window', async () => {
+		const url = 'https://example.com/flaky'
+		await clearCache(url)
+		FetchMock.route(url, () => FetchMock.status(500))
+
+		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })
+
+		const callsAfterFirst = FetchMock.callCount()
+
+		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })
+
+		expect(FetchMock.callCount()).toBe(callsAfterFirst)
+	})
+})
+
+describe('robots.txt', () => {
+
+	test('refuses a disallowed url and records it', async () => {
+		const url = 'https://example.com/no-bots/page'
+		await clearCache(url)
+
+		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 }) // robots ON
+
+		FetchMock.route('https://example.com/robots.txt', () => new Response(
+			'User-agent: *\nDisallow: /no-bots', { status: 200, headers: { 'content-type': 'text/plain' } }
+		))
+		FetchMock.route(url, () => page('Should Not Be Fetched'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect(FetchMock.callsTo(url)).toHaveLength(0)
+
+		const cached = await ScrapeCache.get(url)
+		expect(cached.fetch_state).toBe('blocked')
+	})
+
+	test('proceeds when robots.txt allows the path', async () => {
+		const url = 'https://example.com/allowed/page'
+		await clearCache(url)
+
+		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 })
+
+		FetchMock.route('https://example.com/robots.txt', () => new Response(
+			'User-agent: *\nDisallow: /private', { status: 200, headers: { 'content-type': 'text/plain' } }
+		))
+		FetchMock.route(url, () => page('Allowed Page'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })
+
+		expect((await attachmentRow(attachmentId)).text).toContain('Allowed Page')
+	})
+})
+
+describe('crash recovery', () => {
+
+	test('re-enqueues rows left pending by a restart', async () => {
+		const url = 'https://example.com/recovered'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Recovered Page'))
+
+		// Simulate a row that was enqueued but never processed.
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+
+		const recovery = await ScrapeQueue.recoverPending()
+		expect(recovery.requeued).toBeGreaterThan(0)
+
+		await ScrapeQueue.drain()
+
+		const row = await attachmentRow(attachmentId)
+		expect(row.text).toContain('Recovered Page')
+		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('resets rows stuck mid-fetch so they are retried', async () => {
+		const url = 'https://example.com/stuck'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Unstuck Page'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
+		// Pretend the process died while this one was in flight.
+		await db.promise().query('UPDATE attachment SET scrape_state = ? WHERE id = ?',
+			[ScrapeQueue.STATE.FETCHING, attachmentId])
+
+		await ScrapeQueue.recoverPending()
+		await ScrapeQueue.drain()
+
+		expect((await attachmentRow(attachmentId)).scrape_state).toBe(ScrapeQueue.STATE.DONE)
+	})
+
+	test('does not sweep up long-settled rows', async () => {
+		// Rows default to DONE, which is what keeps the pre-existing ~1400
+		// attachment rows out of the recovery queue on first boot.
+		const [rows] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE scrape_state = ? AND user_id = ?',
+			[ScrapeQueue.STATE.PENDING, userId]
+		)
+		const recovery = await ScrapeQueue.recoverPending()
+		expect(recovery.requeued).toBeLessThanOrEqual(rows[0].n + 1)
+	})
+})

server/tests/support/fetchMock.js

@@ -0,0 +1,127 @@
+//
+// Test harness for the outbound HTTP layer.
+//
+// Responses are built from the REAL Response/ReadableStream globals rather than
+// hand-rolled stubs, so the streaming size cap, the charset decoder and the
+// body-cancel paths in HttpFetch are genuinely exercised instead of being
+// stubbed around.
+//
+
+let FetchMock = module.exports = {}
+
+let originalFetch = null
+let routes = []
+let callLog = []
+
+FetchMock.install = () => {
+	originalFetch = global.fetch
+	routes = []
+	callLog = []
+
+	global.fetch = jest.fn(async (url, options = {}) => {
+		const href = typeof url === 'string' ? url : url.href
+		callLog.push({ url: href, headers: options.headers || {}, options })
+
+		const route = routes.find(entry => entry.match(href))
+		if (!route) {
+			throw Object.assign(new Error(`No route for ${href}`), { code: 'ENOTFOUND' })
+		}
+
+		// A hanging route must still honour the abort signal, or the test would
+		// wait forever instead of proving the AbortController fires.
+		if (options.signal) {
+			return await Promise.race([
+				Promise.resolve(route.responder(href, options)),
+				abortPromise(options.signal),
+			])
+		}
+
+		return route.responder(href, options)
+	})
+}
+
+const abortPromise = (signal) => new Promise((resolve, reject) => {
+	if (signal.aborted) {
+		return reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
+	}
+	signal.addEventListener('abort', () => {
+		reject(Object.assign(new Error('Aborted'), { name: 'AbortError' }))
+	}, { once: true })
+})
+
+FetchMock.restore = () => {
+	if (originalFetch) { global.fetch = originalFetch } else { delete global.fetch }
+	originalFetch = null
+	routes = []
+	callLog = []
+}
+
+//
+// route('https://example.com/', responder) or route(/example/, responder).
+// Responders are called with (url, options) and may be functions returning a
+// Response, so a route can answer differently on each call.
+//
+FetchMock.route = (pattern, responder) => {
+	const match = pattern instanceof RegExp
+		? (href) => pattern.test(href)
+		: (href) => href === pattern
+
+	routes.push({ match, responder: typeof responder === 'function' ? responder : () => responder })
+}
+
+// Answer with a counter so "retried exactly N times" is assertable.
+FetchMock.routeSequence = (pattern, responses) => {
+	let index = 0
+	FetchMock.route(pattern, () => {
+		const response = responses[Math.min(index, responses.length - 1)]
+		index++
+		return typeof response === 'function' ? response() : response
+	})
+}
+
+FetchMock.html = (body, { status = 200, headers = {}, contentType = 'text/html; charset=utf-8' } = {}) => {
+	return new Response(body, {
+		status,
+		headers: { 'content-type': contentType, ...headers },
+	})
+}
+
+// Raw bytes, for the charset tests where the encoding is the point.
+FetchMock.buffer = (buffer, { status = 200, contentType = 'text/html', headers = {} } = {}) => {
+	return new Response(buffer, {
+		status,
+		headers: { 'content-type': contentType, ...headers },
+	})
+}
+
+FetchMock.status = (status, { headers = {}, body = '' } = {}) => {
+	return new Response(body || null, { status, headers })
+}
+
+FetchMock.redirect = (location, status = 302) => {
+	return new Response(null, { status, headers: { location } })
+}
+
+//
+// A body that never ends. Proves the size cap aborts mid-stream rather than
+// buffering until the process dies.
+//
+FetchMock.oversized = (contentType = 'text/html') => {
+	const chunk = Buffer.alloc(64 * 1024, 0x61)
+	const stream = new ReadableStream({
+		pull(controller) { controller.enqueue(new Uint8Array(chunk)) },
+	})
+	return new Response(stream, { status: 200, headers: { 'content-type': contentType } })
+}
+
+// Never settles - the AbortController is the only way out.
+FetchMock.hang = () => new Promise(() => {})
+
+FetchMock.networkError = (code = 'ECONNRESET') => () => {
+	throw Object.assign(new Error(`socket ${code}`), { code })
+}
+
+FetchMock.calls = () => callLog.slice()
+FetchMock.callsTo = (pattern) => callLog.filter(entry =>
+	pattern instanceof RegExp ? pattern.test(entry.url) : entry.url === pattern)
+FetchMock.callCount = () => callLog.length