Adding scrape files for testing

server/tests/helpers/HostQueue.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const HostQueue = require('@helpers/HostQueue')
+
+//
+// A controllable clock that also drives the queue's own timers, so advancing
+// time is instant and spacing assertions are exact rather than flaky.
+//
+const makeClock = () => {
+	let t = 1000
+	let nextId = 1
+	const timers = new Map()
+
+	return {
+		now: () => t,
+		setTimeoutFn: (fn, ms) => {
+			const id = nextId++
+			timers.set(id, { fn, dueAt: t + ms })
+			return id
+		},
+		clearTimeoutFn: (id) => { timers.delete(id) },
+		advance: (ms) => {
+			t += ms
+			for (const [id, timer] of [...timers.entries()]) {
+				if (timer.dueAt <= t) {
+					timers.delete(id)
+					timer.fn()
+				}
+			}
+		},
+	}
+}
+
+const wireClock = (clock) => ({
+	now: clock.now,
+	setTimeoutFn: clock.setTimeoutFn,
+	clearTimeoutFn: clock.clearTimeoutFn,
+})
+
+const deferred = () => {
+	let resolve = null
+	let reject = null
+	const promise = new Promise((res, rej) => { resolve = res; reject = rej })
+	return { promise, resolve, reject }
+}
+
+describe('global concurrency', () => {
+
+	test('never exceeds the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = Array.from({ length: 6 }, (unused, i) => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(`host${i}.com`, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+				return i
+			})
+		})
+
+		// Let the first batch start.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(2)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+
+		expect(peak).toBe(2)
+	})
+
+	test('runs different hosts in parallel up to the cap', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+		const gates = []
+
+		const jobs = ['a.com', 'b.com', 'c.com'].map(host => {
+			const gate = deferred()
+			gates.push(gate)
+			return queue.schedule(host, async () => {
+				active++
+				peak = Math.max(peak, active)
+				await gate.promise
+				active--
+			})
+		})
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(peak).toBe(3)
+
+		gates.forEach(gate => gate.resolve())
+		await Promise.all(jobs)
+	})
+})
+
+describe('per-host serialisation', () => {
+
+	test('runs only one request per host at a time', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 4, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		let active = 0
+		let peak = 0
+
+		const jobs = [1, 2, 3].map(() => queue.schedule('same.com', async () => {
+			active++
+			peak = Math.max(peak, active)
+			await new Promise(resolve => setImmediate(resolve))
+			active--
+		}))
+
+		await Promise.all(jobs)
+
+		expect(peak).toBe(1)
+	})
+
+	test('preserves FIFO order within a host', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+		const order = []
+
+		await Promise.all([1, 2, 3, 4].map(n =>
+			queue.schedule('same.com', async () => { order.push(n) })
+		))
+
+		expect(order).toEqual([1, 2, 3, 4])
+	})
+})
+
+describe('politeness spacing', () => {
+
+	test('leaves a gap between two requests to the same host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 1000,
+			maxSpacingMs: 1000,
+			...wireClock(clock),
+		})
+
+		const startedAt = []
+
+		const first = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+		await first
+
+		const second = queue.schedule('same.com', async () => { startedAt.push(clock.now()) })
+
+		// The second job must not have run yet - the host is spaced out.
+		await new Promise(resolve => setImmediate(resolve))
+		expect(startedAt).toHaveLength(1)
+
+		clock.advance(1000)
+		await second
+
+		expect(startedAt).toHaveLength(2)
+		expect(startedAt[1] - startedAt[0]).toBeGreaterThanOrEqual(1000)
+	})
+
+	test('spacing on one host does not delay a different host', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({
+			globalConcurrency: 4,
+			minSpacingMs: 5000,
+			maxSpacingMs: 5000,
+			...wireClock(clock),
+		})
+
+		await queue.schedule('slow.com', async () => {})
+
+		// slow.com is now parked for 5s. other.com must be unaffected.
+		const ran = []
+		await queue.schedule('other.com', async () => { ran.push('other') })
+
+		expect(ran).toEqual(['other'])
+	})
+
+	test('honours an explicit spacing override (robots Crawl-delay)', async () => {
+		const clock = makeClock()
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0, ...wireClock(clock) })
+
+		await queue.schedule('slow.com', async () => {}, { spacingMs: 3000 })
+
+		const ran = []
+		const second = queue.schedule('slow.com', async () => { ran.push(1) })
+
+		await new Promise(resolve => setImmediate(resolve))
+		expect(ran).toEqual([])
+
+		clock.advance(3000)
+		await second
+		expect(ran).toEqual([1])
+	})
+})
+
+describe('error handling', () => {
+
+	test('a rejecting task does not wedge the queue', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', async () => { throw new Error('boom') }))
+			.rejects.toThrow('boom')
+
+		await expect(queue.schedule('a.com', async () => 'fine')).resolves.toBe('fine')
+	})
+
+	test('a task that throws synchronously is caught', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 0, maxSpacingMs: 0 })
+
+		await expect(queue.schedule('a.com', () => { throw new Error('sync boom') }))
+			.rejects.toThrow('sync boom')
+
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('one failing host does not block others', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const results = await Promise.allSettled([
+			queue.schedule('bad.com', async () => { throw new Error('nope') }),
+			queue.schedule('good.com', async () => 'ok'),
+		])
+
+		expect(results[0].status).toBe('rejected')
+		expect(results[1].value).toBe('ok')
+	})
+})
+
+describe('drain and stop', () => {
+
+	test('drain resolves when everything has finished', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 2, minSpacingMs: 0, maxSpacingMs: 0 })
+		const done = []
+
+		;[1, 2, 3].forEach(n => queue.schedule(`h${n}.com`, async () => { done.push(n) }).catch(() => {}))
+
+		await queue.drain()
+
+		expect(done).toHaveLength(3)
+		expect(queue.size()).toBe(0)
+		expect(queue.inflight()).toBe(0)
+	})
+
+	test('drain resolves immediately on an idle queue', async () => {
+		const queue = new HostQueue.Queue()
+		await expect(queue.drain()).resolves.toBeUndefined()
+	})
+
+	test('stop rejects queued work and refuses new work', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const queued = queue.schedule('b.com', async () => 'never')
+
+		await new Promise(resolve => setImmediate(resolve))
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+		await expect(queue.schedule('c.com', async () => 'x')).rejects.toThrow('stopped')
+
+		gate.resolve('done')
+		await expect(first).resolves.toBe('done')
+	})
+})
+
+describe('event loop', () => {
+
+	//
+	// Regression: the politeness timer used to be unref'd, so if nothing else
+	// held the event loop open Node exited and queued jobs were abandoned with
+	// their promises never settling.
+	//
+	test('a spaced job still resolves with nothing else holding the loop open', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 25, maxSpacingMs: 25 })
+
+		await queue.schedule('same.com', async () => 'first')
+
+		// Only the internal timer can resume this one.
+		await expect(queue.schedule('same.com', async () => 'second')).resolves.toBe('second')
+	})
+
+	test('stop() clears the pending timer so shutdown is not blocked', async () => {
+		const queue = new HostQueue.Queue({ minSpacingMs: 10000, maxSpacingMs: 10000 })
+
+		await queue.schedule('same.com', async () => 'first')
+		const queued = queue.schedule('same.com', async () => 'never')
+
+		queue.stop()
+
+		await expect(queued).rejects.toThrow('stopped')
+	})
+})
+
+describe('stats', () => {
+
+	test('reports pending, running and host counts', async () => {
+		const queue = new HostQueue.Queue({ globalConcurrency: 1, minSpacingMs: 0, maxSpacingMs: 0 })
+
+		const gate = deferred()
+		const first = queue.schedule('a.com', () => gate.promise)
+		const second = queue.schedule('b.com', async () => {})
+
+		await new Promise(resolve => setImmediate(resolve))
+
+		const stats = queue.stats()
+		expect(stats.running).toBe(1)
+		expect(stats.pending).toBe(1)
+
+		gate.resolve()
+		await Promise.all([first, second])
+	})
+})

server/tests/helpers/HttpFetch.test.js

@@ -0,0 +1,479 @@
+require('module-alias/register')
+
+const HttpFetch = require('@helpers/HttpFetch')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+beforeEach(() => {
+	FetchMock.install()
+	// Every hostname in these tests resolves to one public address, so the
+	// safety gate passes and we are testing HttpFetch rather than DNS.
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+	// No real wall-time in backoff.
+	HttpFetch.__setSleep(() => Promise.resolve())
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	HttpFetch.__resetSleep()
+})
+
+describe('fetchDocument happy path', () => {
+
+	test('returns decoded html, status and final url', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<title>Hello</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.status).toBe(200)
+		expect(result.html).toContain('<title>Hello</title>')
+		expect(result.finalUrl).toBe('https://example.com/')
+		expect(result.notModified).toBe(false)
+	})
+
+	test('surfaces etag and last-modified for later conditional requests', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>', {
+			headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.etag).toBe('W/"abc"')
+		expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+})
+
+describe('fetchDocument browser identity', () => {
+
+	test('sends a complete, coherent header set and never claims to be Googlebot', async () => {
+		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/')
+		const headers = FetchMock.calls()[0].headers
+
+		expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/)
+		expect(headers['User-Agent']).not.toMatch(/Googlebot/i)
+		expect(headers['Accept']).toContain('text/html')
+		expect(headers['Accept-Language']).toBeTruthy()
+		expect(headers['Accept-Encoding']).toContain('gzip')
+	})
+
+	test('gives the same host the same profile across calls', async () => {
+		FetchMock.route(/example\.com/, () => FetchMock.html('<p>x</p>'))
+
+		const first = await HttpFetch.fetchDocument('https://example.com/a')
+		const second = await HttpFetch.fetchDocument('https://example.com/b')
+
+		expect(first.profileId).toBe(second.profileId)
+	})
+
+	test('pairs sec-ch-ua only with Chromium profiles', () => {
+		const chrome = BrowserProfiles.profileById('chrome-win')
+		const firefox = BrowserProfiles.profileById('firefox-win')
+		const safari = BrowserProfiles.profileById('safari-mac')
+
+		const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' })
+		const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' })
+		const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' })
+
+		expect(chromeHeaders['sec-ch-ua']).toBeTruthy()
+		expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Firefox sends Sec-Fetch-* but no client hints.
+		expect(firefoxHeaders['sec-ch-ua']).toBeUndefined()
+		expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate')
+
+		// Safari sends neither.
+		expect(safariHeaders['sec-ch-ua']).toBeUndefined()
+		expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined()
+	})
+
+	test('never pairs a chrome UA with a firefox accept header', () => {
+		BrowserProfiles.PROFILES.forEach(profile => {
+			if (profile.family === 'chrome') {
+				expect(profile.ua).toMatch(/Chrome\//)
+				expect(profile.accept).toContain('application/signed-exchange')
+			}
+			if (profile.family === 'firefox') {
+				expect(profile.ua).toMatch(/Firefox\//)
+				expect(profile.accept).not.toContain('signed-exchange')
+			}
+		})
+	})
+})
+
+describe('fetchDocument redirects', () => {
+
+	test('follows a single hop and reports the final url', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301))
+		FetchMock.route('https://example.com/b', () => FetchMock.html('<title>B</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+
+		expect(result.finalUrl).toBe('https://example.com/b')
+		expect(result.html).toContain('B')
+	})
+
+	test('resolves a relative Location header', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('/c', 302))
+		FetchMock.route('https://example.com/c', () => FetchMock.html('<title>C</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/a')
+		expect(result.finalUrl).toBe('https://example.com/c')
+	})
+
+	test('detects a redirect loop', async () => {
+		FetchMock.route('https://example.com/a', () => FetchMock.redirect('https://example.com/b', 302))
+		FetchMock.route('https://example.com/b', () => FetchMock.redirect('https://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'redirect_loop' })
+	})
+
+	test('gives up past the redirect cap', async () => {
+		let n = 0
+		FetchMock.route(/https:\/\/example\.com\/hop/, () => FetchMock.redirect(`https://example.com/hop${++n}`, 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/hop'))
+			.rejects.toMatchObject({ kind: 'too_many_redirects' })
+	})
+
+	// The SSRF case that matters: the first URL is perfectly public.
+	test('blocks a redirect pointing at loopback', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://127.0.0.1:3000/admin', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('blocks a redirect pointing at the cloud metadata endpoint', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://169.254.169.254/latest/meta-data/', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+
+	test('refuses an https to http downgrade', async () => {
+		FetchMock.route('https://example.com/a', FetchMock.redirect('http://example.com/a', 302))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/a'))
+			.rejects.toMatchObject({ kind: 'blocked' })
+	})
+})
+
+describe('fetchDocument retries', () => {
+
+	test('retries a 500 and succeeds', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(500),
+			FetchMock.html('<title>Recovered</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('Recovered')
+		expect(FetchMock.callCount()).toBe(2)
+	})
+
+	test('gives up after exhausting retries', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(503))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 503 })
+
+		// 1 initial attempt + config.retries
+		expect(FetchMock.callCount()).toBe(3)
+	})
+
+	test('does NOT retry a 404 - it is an answer, not a failure', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('does NOT retry a 403', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(403))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/')).rejects.toMatchObject({ status: 403 })
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('honours a short Retry-After on a 429', async () => {
+		const waits = []
+		HttpFetch.__setSleep((ms) => { waits.push(ms); return Promise.resolve() })
+
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.status(429, { headers: { 'retry-after': '2' } }),
+			FetchMock.html('<title>OK</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('OK')
+		expect(waits[0]).toBe(2000)
+	})
+
+	test('refuses to sit on an hour-long Retry-After', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(429, { headers: { 'retry-after': '3600' } }))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/'))
+			.rejects.toMatchObject({ kind: 'http', status: 429 })
+
+		// Surfaced for the cache to park the URL rather than retried.
+		expect(FetchMock.callCount()).toBe(1)
+	})
+
+	test('retries a network error', async () => {
+		FetchMock.routeSequence('https://example.com/', [
+			FetchMock.networkError('ECONNRESET'),
+			FetchMock.html('<title>Up</title>'),
+		])
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('Up')
+	})
+})
+
+describe('fetchDocument limits and gates', () => {
+
+	test('times out on a server that never responds', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.hang())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { timeoutMs: 60, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'timeout' })
+	})
+
+	test('aborts a body that exceeds the cap mid-stream', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.oversized())
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { maxBytes: 128 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects an oversized declared content-length without reading the body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>small</p>', {
+			headers: { 'content-length': String(50 * 1024 * 1024) },
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('refuses a non-html content type before parsing', async () => {
+		FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'application/pdf',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('refuses a binary masquerading behind an image content type', async () => {
+		FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), {
+			contentType: 'image/png',
+		}))
+
+		await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+})
+
+describe('fetchDocument charset handling', () => {
+
+	test('decodes windows-1252 declared in the content-type header', async () => {
+		// 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8.
+		const body = Buffer.concat([
+			Buffer.from('<title>', 'latin1'),
+			Buffer.from([0x93]),
+			Buffer.from('caf', 'latin1'),
+			Buffer.from([0xe9, 0x94]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=windows-1252',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+
+		expect(result.html).toContain('“café”')
+		expect(result.charset).toBe('windows-1252')
+	})
+
+	test('falls back to a meta charset when the header says nothing', async () => {
+		const body = Buffer.concat([
+			Buffer.from('<meta charset="windows-1252"><title>caf', 'latin1'),
+			Buffer.from([0xe9]),
+			Buffer.from('</title>', 'latin1'),
+		])
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, { contentType: 'text/html' }))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('header charset wins over the meta tag', async () => {
+		const body = Buffer.from('<meta charset="windows-1252"><title>café</title>', 'utf8')
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
+			contentType: 'text/html; charset=utf-8',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('café')
+	})
+
+	test('survives a bogus charset label', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.buffer(Buffer.from('<p>ok</p>'), {
+			contentType: 'text/html; charset=not-a-real-charset',
+		}))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/')
+		expect(result.html).toContain('ok')
+	})
+})
+
+describe('fetchDocument conditional requests', () => {
+
+	test('sends If-None-Match and If-Modified-Since when given', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.html('<p>x</p>'))
+
+		await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' },
+		})
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['If-None-Match']).toBe('W/"v1"')
+		expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
+	})
+
+	test('reports a 304 as notModified with no body', async () => {
+		FetchMock.route('https://example.com/', () => FetchMock.status(304))
+
+		const result = await HttpFetch.fetchDocument('https://example.com/', {
+			conditional: { etag: 'W/"v1"' },
+		})
+
+		expect(result.notModified).toBe(true)
+		expect(result.status).toBe(304)
+		expect(result.html).toBeNull()
+	})
+})
+
+describe('fetchBinary', () => {
+
+	const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+
+	test('returns the buffer for a real image', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		const result = await HttpFetch.fetchBinary('https://example.com/i.png')
+
+		expect(result.mime).toBe('image/png')
+		expect(result.buffer.equals(pngMagic)).toBe(true)
+	})
+
+	// The old code wrote a 404 HTML error page to disk and handed it to gm.
+	test('refuses an html error page served where an image was expected', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.html('<h1>404</h1>'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('propagates a 404 status rather than writing anything', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.status(404))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'http', status: 404 })
+	})
+
+	test('enforces the image size cap', async () => {
+		FetchMock.route('https://example.com/i.png', () => FetchMock.oversized('image/png'))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { maxBytes: 64 * 1024, retries: 0 }))
+			.rejects.toMatchObject({ kind: 'too_large' })
+	})
+
+	test('rejects svg by default', async () => {
+		FetchMock.route('https://example.com/i.svg', () => FetchMock.buffer(Buffer.from('<svg/>'), {
+			contentType: 'image/svg+xml',
+		}))
+
+		await expect(HttpFetch.fetchBinary('https://example.com/i.svg', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'bad_content_type' })
+	})
+
+	test('sends image-appropriate fetch metadata and a page referer', async () => {
+		FetchMock.route('https://cdn.example.net/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))
+
+		await HttpFetch.fetchBinary('https://cdn.example.net/i.png', { pageUrl: 'https://example.com/post' })
+
+		const headers = FetchMock.calls()[0].headers
+		expect(headers['Referer']).toBe('https://example.com/post')
+		expect(headers['Accept']).toContain('image/')
+		if (headers['Sec-Fetch-Dest']) {
+			expect(headers['Sec-Fetch-Dest']).toBe('image')
+			expect(headers['Sec-Fetch-Site']).toBe('cross-site')
+		}
+	})
+})
+
+describe('transient vs terminal safety failures', () => {
+
+	// A private address is a permanent refusal; a DNS failure means the host
+	// may just be down. Classifying both as "blocked" parked transient
+	// failures for the full cache TTL.
+	test('classifies a private address as blocked', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' })
+	})
+
+	test('classifies a DNS failure as network, not blocked', async () => {
+		dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' })
+	})
+
+	test('retries a DNS failure, since it may be transient', async () => {
+		let attempts = 0
+		dnsSpy.mockImplementation(async () => {
+			attempts++
+			if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) }
+			return [{ address: '93.184.216.34', family: 4 }]
+		})
+		FetchMock.route('https://flaky.example/', () => FetchMock.html('<title>Back up</title>'))
+
+		const result = await HttpFetch.fetchDocument('https://flaky.example/')
+		expect(result.html).toContain('Back up')
+		expect(attempts).toBe(2)
+	})
+
+	test('does NOT retry a private address', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' })
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})
+
+describe('safety gate on the initial url', () => {
+
+	test('refuses to open a connection to a private address at all', async () => {
+		FetchMock.route(/.*/, () => FetchMock.html('<p>should never be reached</p>'))
+
+		await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 }))
+			.rejects.toMatchObject({ kind: 'blocked' })
+
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})

server/tests/helpers/Robots.test.js

@@ -0,0 +1,173 @@
+require('module-alias/register')
+
+const Robots = require('@helpers/Robots')
+const config = require('@config/scrape')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+const UA = config.robotsUaToken
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	Robots.__resetCache()
+})
+
+const serveRobots = (body, origin = 'https://example.com') => {
+	FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
+		status: 200,
+		headers: { 'content-type': 'text/plain' },
+	}))
+}
+
+describe('Robots.parse', () => {
+
+	test('groups consecutive user-agent lines into one rule block', () => {
+		const rules = Robots.parse([
+			'User-agent: alpha',
+			'User-agent: beta',
+			'Disallow: /private',
+		].join('\n'))
+
+		expect(rules.groups).toHaveLength(1)
+		expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
+	})
+
+	test('ignores comments and blank lines', () => {
+		const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
+	})
+
+	test('treats an empty Disallow as no restriction', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow:')
+		expect(rules.groups[0].rules).toHaveLength(0)
+	})
+
+	test('parses crawl-delay', () => {
+		const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
+		expect(rules.groups[0].crawlDelayMs).toBe(2500)
+	})
+
+	test('an empty file allows everything', () => {
+		expect(Robots.parse('').allowAll).toBe(true)
+		expect(Robots.parse(null).allowAll).toBe(true)
+	})
+})
+
+describe('Robots.isPathAllowed', () => {
+
+	test('respects a simple disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /private')
+		expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
+	})
+
+	test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
+		expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
+	})
+
+	test('Allow beats Disallow at equal specificity', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
+		expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
+	})
+
+	test('supports the * wildcard', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
+		expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
+	})
+
+	test('supports the $ anchor', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /x$')
+		expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
+	})
+
+	test('a named group beats the wildcard group even when listed later', () => {
+		const rules = Robots.parse([
+			'User-agent: *',
+			'Disallow: /',
+			'',
+			`User-agent: ${UA}`,
+			'Disallow:',
+		].join('\n'))
+
+		expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
+		expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
+	})
+})
+
+describe('Robots.isAllowed over the network', () => {
+
+	test('refuses a disallowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
+	})
+
+	test('permits an allowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
+	})
+
+	// Fail-open: a missing robots.txt is not a refusal.
+	test('allows everything when robots.txt 404s', async () => {
+		FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('allows everything when robots.txt cannot be reached', async () => {
+		FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('caches per origin - two queries, one network call', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+
+		await Robots.isAllowed('https://example.com/a')
+		await Robots.isAllowed('https://example.com/secret/b')
+
+		expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
+	})
+
+	test('keeps origins separate', async () => {
+		serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
+		serveRobots('User-agent: *\nDisallow:', 'https://open.com')
+
+		await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
+		await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
+	})
+
+	test('returns false for an unparseable url rather than throwing', async () => {
+		await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
+	})
+})
+
+describe('Robots.crawlDelayMs', () => {
+
+	test('reports the declared delay', async () => {
+		serveRobots('User-agent: *\nCrawl-delay: 5')
+		await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
+	})
+
+	test('is null when none is declared', async () => {
+		serveRobots('User-agent: *\nDisallow: /x')
+		await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
+	})
+
+	test('flags a delay beyond the cap instead of parking the job forever', async () => {
+		expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
+		expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
+		expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
+	})
+})

server/tests/helpers/SiteScrape.test.js

@@ -0,0 +1,318 @@
+require('module-alias/register')
+
+const fs = require('fs')
+const path = require('path')
+const cheerio = require('cheerio')
+
+const SiteScrape = require('@helpers/SiteScrape')
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'html')
+
+// Fixtures are frozen snapshots - see fixtures/html/README.md. No network here.
+const load = (name) => cheerio.load(fs.readFileSync(path.join(FIXTURES, name), 'utf8'))
+
+const PAGE_URL = 'https://example.com/blog/2026/post'
+
+describe('getTitle', () => {
+
+	test('prefers og:title over the title element', () => {
+		expect(SiteScrape.getTitle(load('og-complete.html'))).toBe('The Real Title')
+	})
+
+	test('falls back to twitter:title', () => {
+		expect(SiteScrape.getTitle(load('twitter-only.html'))).toBe('Twitter Card Title')
+	})
+
+	test('falls back to JSON-LD headline', () => {
+		expect(SiteScrape.getTitle(load('jsonld-article.html'))).toBe('JSON-LD Headline')
+	})
+
+	test('falls back to the title element and collapses whitespace', () => {
+		expect(SiteScrape.getTitle(load('title-only.html'))).toBe('Just A Title')
+	})
+
+	test('falls back to an h1 when there is no title element', () => {
+		expect(SiteScrape.getTitle(load('h1-fallback.html'))).toBe('Heading Is All We Have')
+	})
+
+	test('returns empty string for a page with nothing', () => {
+		expect(SiteScrape.getTitle(load('no-metadata.html'))).toBe('')
+	})
+})
+
+describe('getDescription', () => {
+
+	test('prefers og:description', () => {
+		expect(SiteScrape.getDescription(load('og-complete.html')))
+			.toBe('A proper social description of the page.')
+	})
+
+	test('falls back to twitter:description', () => {
+		expect(SiteScrape.getDescription(load('twitter-only.html'))).toBe('Twitter card description.')
+	})
+
+	test('falls back to JSON-LD description', () => {
+		expect(SiteScrape.getDescription(load('jsonld-article.html'))).toBe('Described in structured data.')
+	})
+
+	test('is empty when no description exists', () => {
+		expect(SiteScrape.getDescription(load('title-only.html'))).toBe('')
+	})
+})
+
+describe('getJsonLd', () => {
+
+	test('reads a plain block', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-article.html')).headline).toBe('JSON-LD Headline')
+	})
+
+	test('digs the content node out of an @graph', () => {
+		expect(SiteScrape.getJsonLd(load('jsonld-graph.html')).headline).toBe('Graph Headline')
+	})
+
+	// Malformed JSON-LD is extremely common and must never throw.
+	test('survives malformed JSON without throwing', () => {
+		expect(() => SiteScrape.getJsonLd(load('jsonld-malformed.html'))).not.toThrow()
+		expect(SiteScrape.getTitle(load('jsonld-malformed.html'))).toBe('Survived Bad JSON')
+	})
+
+	test('is null when absent', () => {
+		expect(SiteScrape.getJsonLd(load('title-only.html'))).toBeNull()
+	})
+})
+
+describe('getCanonicalUrl', () => {
+
+	test('uses link rel=canonical', () => {
+		expect(SiteScrape.getCanonicalUrl(load('og-complete.html'), PAGE_URL))
+			.toBe('https://example.com/canonical-post')
+	})
+
+	test('falls back to the requested url', () => {
+		expect(SiteScrape.getCanonicalUrl(load('title-only.html'), PAGE_URL)).toBe(PAGE_URL)
+	})
+})
+
+describe('getDisplayImage', () => {
+
+	test('prefers og:image', () => {
+		expect(SiteScrape.getDisplayImage(load('og-complete.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.jpg')
+	})
+
+	test('falls back to twitter:image', () => {
+		expect(SiteScrape.getDisplayImage(load('twitter-only.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/twitter.png')
+	})
+
+	test('falls back to the JSON-LD image, taking the first of an array', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-article.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/ld-primary.jpg')
+	})
+
+	test('reads an object-form JSON-LD image', () => {
+		expect(SiteScrape.getDisplayImage(load('jsonld-graph.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/graph.png')
+	})
+
+	// Resolution against the canonical base, not string concatenation.
+	test('resolves a relative path against the canonical url', () => {
+		expect(SiteScrape.getDisplayImage(load('relative-images.html'), PAGE_URL))
+			.toBe('https://example.com/blog/images/hero.png')
+	})
+
+	test('resolves a root-relative icon', () => {
+		expect(SiteScrape.getDisplayImage(load('root-relative-icon.html'), PAGE_URL))
+			.toBe('https://example.com/favicon.ico')
+	})
+
+	test('resolves a protocol-relative url', () => {
+		expect(SiteScrape.getDisplayImage(load('protocol-relative-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/proto.jpg')
+	})
+
+	// The old substring filter dropped all of these.
+	test('accepts .webp', () => {
+		expect(SiteScrape.getDisplayImage(load('webp-og-image.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/hero.webp')
+	})
+
+	test('accepts an extensionless og:image endpoint', () => {
+		expect(SiteScrape.getDisplayImage(load('extensionless-og-image.html'), PAGE_URL))
+			.toBe('https://example.com/api/og?title=Hello&theme=dark')
+	})
+
+	test('rejects svg while allowSvgThumbnails is off', () => {
+		expect(SiteScrape.getDisplayImage(load('svg-only-image.html'), PAGE_URL)).toBe('')
+	})
+
+	test('picks the largest srcset candidate', () => {
+		expect(SiteScrape.getDisplayImage(load('srcset.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/large.jpg')
+	})
+
+	test('prefers the largest inline image by declared area', () => {
+		expect(SiteScrape.getDisplayImage(load('sized-images.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/big.jpg')
+	})
+
+	test('skips tracking pixels and spacers', () => {
+		expect(SiteScrape.getDisplayImage(load('tracking-pixels.html'), PAGE_URL))
+			.toBe('https://cdn.example.com/real-photo.jpg')
+	})
+
+	// Regression: the old code read element.attribs.src directly and threw.
+	test('does not throw on img tags with no src, and finds a data-src', () => {
+		const $ = load('imgs-missing-src.html')
+		expect(() => SiteScrape.getDisplayImage($, PAGE_URL)).not.toThrow()
+		expect(SiteScrape.getDisplayImage($, PAGE_URL)).toBe('https://cdn.example.com/lazy.jpg')
+	})
+
+	test('prefers a large apple-touch-icon over a 16x16 favicon', () => {
+		expect(SiteScrape.getDisplayImage(load('apple-touch-icon.html'), PAGE_URL))
+			.toBe('https://example.com/touch-180.png')
+	})
+
+	test('returns empty string when there is no usable image', () => {
+		expect(SiteScrape.getDisplayImage(load('no-metadata.html'), PAGE_URL)).toBe('')
+	})
+
+	test('survives malformed markup', () => {
+		expect(() => SiteScrape.getDisplayImage(load('malformed-unclosed.html'), PAGE_URL)).not.toThrow()
+	})
+})
+
+describe('getImageCandidates', () => {
+
+	test('returns an ordered, deduplicated list', () => {
+		const candidates = SiteScrape.getImageCandidates(load('tracking-pixels.html'), PAGE_URL)
+		expect(candidates).toEqual(['https://cdn.example.com/real-photo.jpg'])
+	})
+
+	test('is empty rather than null for a bare page', () => {
+		expect(SiteScrape.getImageCandidates(load('no-metadata.html'), PAGE_URL)).toEqual([])
+	})
+})
+
+describe('getKeywords', () => {
+
+	test('extracts page terms and ignores nav, script and footer chrome', () => {
+		const keywords = SiteScrape.getKeywords(load('keyword-content.html'))
+
+		expect(keywords).toMatch(/^Keywords: /)
+		expect(keywords).toMatch(/beekeeping|beehive|honey/)
+		expect(keywords).not.toMatch(/navigation/)
+		expect(keywords).not.toMatch(/tracking/)
+		expect(keywords).not.toMatch(/footer/)
+	})
+
+	test('returns empty string when there is nothing to count', () => {
+		expect(SiteScrape.getKeywords(load('no-metadata.html'))).toBe('')
+	})
+
+	test('does not log page content', () => {
+		// The old implementation dumped up to 6000 chars of every page.
+		const spy = jest.spyOn(console, 'log').mockImplementation(() => {})
+		SiteScrape.getKeywords(load('keyword-content.html'))
+		expect(spy).not.toHaveBeenCalled()
+		spy.mockRestore()
+	})
+})
+
+describe('getHostName', () => {
+
+	test('returns the https origin', () => {
+		expect(SiteScrape.getHostName('http://www.google.com/search?q=x')).toBe('https://www.google.com')
+	})
+
+	test('returns empty string for a bad url instead of throwing', () => {
+		expect(() => SiteScrape.getHostName('nonsense')).not.toThrow()
+		expect(SiteScrape.getHostName('nonsense')).toBe('')
+	})
+})
+
+describe('buildSearchText', () => {
+
+	test('joins the parts that exist', () => {
+		const text = SiteScrape.buildSearchText({
+			title: 'T', description: 'D', siteName: 'S', keywords: 'Keywords: a, b',
+		})
+		expect(text).toBe('T\nD\nS\nKeywords: a, b')
+	})
+
+	test('skips empty parts', () => {
+		expect(SiteScrape.buildSearchText({ title: 'T', description: '', siteName: '', keywords: '' })).toBe('T')
+	})
+
+	test('caps the length so it cannot bloat attachment.text', () => {
+		const text = SiteScrape.buildSearchText({ title: 'x'.repeat(50000) })
+		expect(text.length).toBeLessThanOrEqual(8000)
+	})
+
+	// Sentinel strings used to land in attachment.text and pollute FULLTEXT.
+	test('never produces the old failure sentinels', () => {
+		const text = SiteScrape.buildSearchText({ title: '', description: '', siteName: '', keywords: '' })
+		expect(text).toBe('')
+		expect(text).not.toContain('No site text')
+		expect(text).not.toContain('Request Timeout')
+	})
+})
+
+describe('parse', () => {
+
+	test('returns everything a link preview needs in one call', () => {
+		const result = SiteScrape.parse(load('og-complete.html'), PAGE_URL)
+
+		expect(result).toMatchObject({
+			title: 'The Real Title',
+			description: 'A proper social description of the page.',
+			siteName: 'Example Journal',
+			canonicalUrl: 'https://example.com/canonical-post',
+			imageUrl: 'https://cdn.example.com/hero.jpg',
+			hostname: 'https://example.com',
+		})
+		expect(result.searchText).toContain('The Real Title')
+		expect(result.searchText).toContain('A proper social description')
+	})
+
+	test('degrades to empty fields rather than throwing on a bare page', () => {
+		const result = SiteScrape.parse(load('no-metadata.html'), PAGE_URL)
+
+		expect(result.title).toBe('')
+		expect(result.imageUrl).toBe('')
+		expect(result.searchText).toBe('example.com')
+	})
+})
+
+//
+// getCleanUrls is deliberately unchanged - it decides which URLs are detected in
+// note text and what string is stored in attachment.url. These tests pin the
+// CURRENT behaviour so the rewrite around it cannot drift link detection.
+//
+describe('getCleanUrls (pinned, must not change)', () => {
+
+	test('finds bare and full urls', () => {
+		expect(SiteScrape.getCleanUrls('see https://example.com/a and www.foo.com here'))
+			.toEqual(['https://example.com/a', 'https://www.foo.com'])
+	})
+
+	test('still forces http to https', () => {
+		expect(SiteScrape.getCleanUrls('http://example.com/x')).toEqual(['https://example.com/x'])
+	})
+
+	test('still prepends https to a bare www host', () => {
+		expect(SiteScrape.getCleanUrls('www.example.com')).toEqual(['https://www.example.com'])
+	})
+
+	test('deduplicates', () => {
+		expect(SiteScrape.getCleanUrls('https://a.com https://a.com')).toEqual(['https://a.com'])
+	})
+
+	test('returns an empty array when there are no urls', () => {
+		expect(SiteScrape.getCleanUrls('just some words')).toEqual([])
+	})
+
+	test('returns an empty array for empty text', () => {
+		expect(SiteScrape.getCleanUrls('')).toEqual([])
+	})
+})

server/tests/helpers/UrlSafety.test.js

@@ -0,0 +1,189 @@
+require('module-alias/register')
+
+const UrlSafety = require('@helpers/UrlSafety')
+
+describe('UrlSafety.isPrivateIp', () => {
+
+	const privateAddresses = [
+		'0.0.0.0', '10.0.0.1', '10.255.255.254', '127.0.0.1', '127.1.1.1',
+		'169.254.1.1', '169.254.169.254',  // cloud metadata endpoint
+		'172.16.0.1', '172.31.255.254', '192.168.1.1', '192.0.0.1', '192.0.2.5',
+		'198.18.0.1', '100.64.0.1', '100.127.255.254', '224.0.0.1', '255.255.255.255',
+		'::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1',
+		'::ffff:127.0.0.1', '::ffff:10.0.0.1', '64:ff9b::127.0.0.1',
+	]
+
+	privateAddresses.forEach(address => {
+		test(`blocks ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(true)
+		})
+	})
+
+	const publicAddresses = ['1.1.1.1', '8.8.8.8', '93.184.216.34', '172.32.0.1', '2606:4700::1111']
+
+	publicAddresses.forEach(address => {
+		test(`allows ${address}`, () => {
+			expect(UrlSafety.isPrivateIp(address)).toBe(false)
+		})
+	})
+
+	test('treats a non-IP string as unsafe', () => {
+		expect(UrlSafety.isPrivateIp('not-an-ip')).toBe(true)
+		expect(UrlSafety.isPrivateIp('')).toBe(true)
+	})
+})
+
+describe('UrlSafety.parse', () => {
+
+	test('rejects non-http schemes', () => {
+		expect(UrlSafety.parse('file:///etc/passwd')).toBeNull()
+		expect(UrlSafety.parse('ftp://example.com/x')).toBeNull()
+		expect(UrlSafety.parse('javascript:alert(1)')).toBeNull()
+		expect(UrlSafety.parse('data:text/html,<h1>x</h1>')).toBeNull()
+	})
+
+	test('rejects garbage and empty input', () => {
+		expect(UrlSafety.parse('')).toBeNull()
+		expect(UrlSafety.parse(null)).toBeNull()
+		expect(UrlSafety.parse('not a url at all')).toBeNull()
+	})
+
+	test('rejects URLs longer than the attachment.url column', () => {
+		const long = 'https://example.com/' + 'a'.repeat(1200)
+		expect(UrlSafety.parse(long)).toBeNull()
+	})
+
+	test('accepts ordinary http and https', () => {
+		expect(UrlSafety.parse('https://example.com/a').href).toBe('https://example.com/a')
+		expect(UrlSafety.parse('http://example.com/a').href).toBe('http://example.com/a')
+	})
+})
+
+describe('UrlSafety.normalizeForKey / urlKey', () => {
+
+	test('strips tracking params so the cache key is stable', () => {
+		const bare = UrlSafety.urlKey('https://example.com/post')
+		const noisy = UrlSafety.urlKey('https://example.com/post?utm_source=x&utm_medium=y&fbclid=z')
+		expect(noisy).toBe(bare)
+	})
+
+	test('keeps meaningful query params', () => {
+		const a = UrlSafety.urlKey('https://example.com/search?q=cats')
+		const b = UrlSafety.urlKey('https://example.com/search?q=dogs')
+		expect(a).not.toBe(b)
+	})
+
+	test('is insensitive to param order, host case, fragment and trailing host dot', () => {
+		const canonical = UrlSafety.urlKey('https://example.com/p?a=1&b=2')
+		expect(UrlSafety.urlKey('https://example.com/p?b=2&a=1')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://EXAMPLE.com/p?a=1&b=2')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com/p?a=1&b=2#section')).toBe(canonical)
+		expect(UrlSafety.urlKey('https://example.com./p?a=1&b=2')).toBe(canonical)
+	})
+
+	test('distinguishes http from https', () => {
+		expect(UrlSafety.urlKey('http://example.com/')).not.toBe(UrlSafety.urlKey('https://example.com/'))
+	})
+
+	test('returns a 64-char hex digest', () => {
+		expect(UrlSafety.urlKey('https://example.com/')).toMatch(/^[0-9a-f]{64}$/)
+	})
+
+	test('returns null for an unusable URL', () => {
+		expect(UrlSafety.urlKey('javascript:alert(1)')).toBeNull()
+	})
+})
+
+describe('UrlSafety.sameResource', () => {
+
+	// This is what stops scanTextForWebsites deleting and re-scraping a row
+	// every time a note is saved.
+	test('treats http/https, www. and trailing slash as the same resource', () => {
+		expect(UrlSafety.sameResource('http://foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://www.foo.com/a', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com/a/', 'https://foo.com/a')).toBe(true)
+		expect(UrlSafety.sameResource('https://foo.com', 'https://foo.com/')).toBe(true)
+	})
+
+	test('does not conflate different paths, hosts or queries', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://foo.com/b')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a', 'https://bar.com/a')).toBe(false)
+		expect(UrlSafety.sameResource('https://foo.com/a?x=1', 'https://foo.com/a?x=2')).toBe(false)
+	})
+
+	test('is false when either side is unusable', () => {
+		expect(UrlSafety.sameResource('https://foo.com/a', 'javascript:x')).toBe(false)
+	})
+})
+
+describe('UrlSafety.assertFetchable', () => {
+
+	const expectRejection = async (url, reason) => {
+		await expect(UrlSafety.assertFetchable(url)).rejects.toMatchObject({ name: 'SafetyError', reason })
+	}
+
+	test('refuses non-http schemes', async () => {
+		await expectRejection('file:///etc/passwd', 'bad_url')
+	})
+
+	test('refuses embedded credentials', async () => {
+		await expectRejection('https://user:[email protected]/', 'credentials_in_url')
+	})
+
+	test('refuses localhost and internal suffixes', async () => {
+		await expectRejection('http://localhost:3000/', 'blocked_host')
+		await expectRejection('http://db.internal/', 'blocked_host')
+		await expectRejection('http://thing.local/', 'blocked_host')
+		await expectRejection('http://box.lan/', 'blocked_host')
+	})
+
+	test('refuses literal private addresses without touching DNS', async () => {
+		await expectRejection('http://127.0.0.1:3000/', 'private_ip')
+		await expectRejection('http://169.254.169.254/latest/meta-data/', 'private_ip')
+		await expectRejection('http://10.1.2.3/', 'private_ip')
+		await expectRejection('http://[::1]:8080/', 'private_ip')
+	})
+
+	test('allows a literal public address', async () => {
+		await expect(UrlSafety.assertFetchable('http://1.1.1.1/')).resolves.toEqual(['1.1.1.1'])
+	})
+
+	test('refuses a hostname that resolves to a private address', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '192.168.0.5', family: 4 }])
+
+		await expectRejection('https://rebind.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('refuses when ANY address in a round-robin record is private', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([
+			{ address: '93.184.216.34', family: 4 },
+			{ address: '127.0.0.1', family: 4 },
+		])
+
+		await expectRejection('https://mixed.example.com/', 'private_ip')
+
+		spy.mockRestore()
+	})
+
+	test('allows a hostname that resolves entirely to public addresses', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+
+		await expect(UrlSafety.assertFetchable('https://example.com/')).resolves.toEqual(['93.184.216.34'])
+
+		spy.mockRestore()
+	})
+
+	test('surfaces a DNS failure as dns_failed', async () => {
+		const dns = require('dns').promises
+		const spy = jest.spyOn(dns, 'lookup').mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
+
+		await expectRejection('https://nonexistent.example/', 'dns_failed')
+
+		spy.mockRestore()
+	})
+})