Adding scrape files for testing

server/helpers/HttpFetch.js

@@ -0,0 +1,429 @@
+//
+// The only module in the server that opens an outbound connection.
+//
+// Everything that made the old scraper unreliable is handled here, in one place:
+// a single AbortController-based timeout (the old code raced request's socket
+// timeout against a manual setTimeout that could fire against a null reference
+// and crash the process), manual redirect following so the SSRF gate runs on
+// every hop, hard size caps enforced while streaming, a Content-Type gate before
+// any body is read, real charset decoding, and retries that only fire for
+// transient failures.
+//
+
+const config = require('@config/scrape')
+const UrlSafety = require('@helpers/UrlSafety')
+const BrowserProfiles = require('@helpers/BrowserProfiles')
+
+let HttpFetch = module.exports = {}
+
+const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']
+
+class FetchError extends Error {
+	constructor(kind, message, extra = {}) {
+		super(message)
+		this.name = 'FetchError'
+		this.kind = kind          // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
+		this.status = extra.status || null
+		this.retryAfterMs = extra.retryAfterMs || null
+		this.url = extra.url || null
+		this.reason = extra.reason || null
+	}
+}
+HttpFetch.FetchError = FetchError
+
+// Injectable so tests don't spend real wall-time in backoff. Far simpler than
+// interleaving fake timers with promises.
+let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+HttpFetch.__setSleep = (fn) => { sleep = fn }
+HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }
+
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])
+
+const isRetryable = (error) => {
+	if (!(error instanceof FetchError)) { return false }
+	if (error.kind === 'network' || error.kind === 'timeout') { return true }
+	if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
+	return false
+}
+
+// Retry-After is either delta-seconds or an HTTP-date.
+const parseRetryAfter = (value) => {
+	if (!value) { return null }
+
+	const seconds = Number(value)
+	if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }
+
+	const when = Date.parse(value)
+	if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }
+
+	return null
+}
+
+const backoffFor = (attempt) => {
+	const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
+	return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
+}
+
+const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
+const mimeOf = (contentType) => contentType.split(';')[0].trim()
+
+const charsetFromContentType = (contentType) => {
+	const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
+	return match ? match[1].toLowerCase() : null
+}
+
+// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
+// byte maps to a character and the regex can't miss.
+const charsetFromMeta = (buffer) => {
+	const head = buffer.subarray(0, 2048).toString('latin1')
+
+	const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
+	if (metaCharset) { return metaCharset[1].toLowerCase() }
+
+	const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
+	if (httpEquiv) { return httpEquiv[1].toLowerCase() }
+
+	return null
+}
+
+const charsetFromBom = (buffer) => {
+	if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
+	if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
+	if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
+	return null
+}
+
+const decodeBody = (buffer, declaredCharset) => {
+	const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'
+
+	try {
+		return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
+	} catch (error) {
+		// Unknown/bogus label - utf-8 with replacement chars beats throwing.
+		return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
+	}
+}
+
+//
+// Read the body with a running byte count, aborting mid-stream once the cap is
+// passed. Content-Length is checked by the caller too, but it is optional and
+// can lie, so this is the cap that actually holds.
+//
+const readCapped = async (response, maxBytes, controller, url) => {
+	if (!response.body) { return Buffer.alloc(0) }
+
+	const chunks = []
+	let total = 0
+
+	for await (const chunk of response.body) {
+		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+		total += buffer.length
+
+		if (total > maxBytes) {
+			try { controller.abort() } catch (error) { /* already done */ }
+			throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
+		}
+
+		chunks.push(buffer)
+	}
+
+	return Buffer.concat(chunks, total)
+}
+
+//
+// One HTTP request, following redirects by hand.
+//
+// Manual redirects are the whole point: fetch's own 'follow' would hop to
+// 127.0.0.1 without ever re-consulting the safety gate.
+//
+const requestOnce = async (startUrl, options) => {
+	const {
+		maxBytes,
+		acceptMimes,
+		headersFor,
+		conditional,
+		timeoutMs,
+		deadline,
+		referer,
+	} = options
+
+	let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
+	const visited = new Set()
+
+	for (let hop = 0; hop <= config.maxRedirects; hop++) {
+		if (visited.has(currentUrl)) {
+			throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
+		}
+		visited.add(currentUrl)
+
+		// Re-validate on every hop. This is the check the old code never had.
+		try {
+			await UrlSafety.assertFetchable(currentUrl)
+		} catch (error) {
+			// A DNS failure is transient - the host may simply be down right
+			// now - so it must not be classified alongside "this address is
+			// private", which is a permanent refusal. Collapsing the two meant
+			// a briefly-unresolvable domain got parked for the full cache TTL
+			// and had its attachment row deleted.
+			const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
+			throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
+		}
+
+		const remaining = deadline - Date.now()
+		if (remaining <= 0) {
+			throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
+		}
+
+		const controller = new AbortController()
+		const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))
+
+		let response = null
+		try {
+			const headers = headersFor(currentUrl, referer)
+
+			if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
+			if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }
+
+			response = await fetch(currentUrl, {
+				method: 'GET',
+				redirect: 'manual',
+				signal: controller.signal,
+				headers,
+			})
+		} catch (error) {
+			clearTimeout(timer)
+
+			if (controller.signal.aborted) {
+				throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
+			}
+			throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
+		}
+
+		// Redirect?
+		if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
+			clearTimeout(timer)
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }
+
+			const location = response.headers.get('location')
+			let next = null
+			try {
+				next = new URL(location, currentUrl)
+			} catch (error) {
+				throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
+			}
+
+			// Never follow https -> http. A downgrade mid-chain is either an
+			// attack or a misconfiguration, and is not worth the preview.
+			if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
+				throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
+			}
+
+			currentUrl = next.href
+			continue
+		}
+
+		clearTimeout(timer)
+
+		if (response.status === 304) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			return { notModified: true, status: 304, finalUrl: currentUrl, response }
+		}
+
+		if (!response.ok) {
+			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
+			throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
+				status: response.status,
+				retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
+				url: currentUrl,
+			})
+		}
+
+		const contentType = contentTypeOf(response)
+		const mime = mimeOf(contentType)
+
+		// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
+		// being handed to cheerio.
+		if (acceptMimes && !acceptMimes(mime)) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
+		}
+
+		const declaredLength = Number(response.headers.get('content-length'))
+		if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+			try { controller.abort() } catch (error) { /* ignore */ }
+			throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
+		}
+
+		const buffer = await readCapped(response, maxBytes, controller, currentUrl)
+
+		return {
+			notModified: false,
+			status: response.status,
+			finalUrl: currentUrl,
+			contentType,
+			mime,
+			buffer,
+			response,
+		}
+	}
+
+	throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
+}
+
+//
+// Retry wrapper. Only transient classes are retried - a 404 is an answer.
+//
+const withRetries = async (attemptFn, { retries, deadline }) => {
+	let lastError = null
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		try {
+			return await attemptFn()
+		} catch (error) {
+			lastError = error
+
+			if (!isRetryable(error) || attempt === retries) { throw error }
+
+			// A server asking us to come back in an hour is not something to
+			// sit on - surface it so the cache can park the URL instead.
+			if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }
+
+			const wait = error.retryAfterMs != null
+				? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
+				: backoffFor(attempt)
+
+			if (Date.now() + wait >= deadline) { throw error }
+
+			await sleep(wait)
+		}
+	}
+
+	throw lastError
+}
+
+//
+// Fetch an HTML document.
+//
+HttpFetch.fetchDocument = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.pageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxHtmlBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes: (mime) => HTML_TYPES.includes(mime),
+		headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
+			url: target,
+			// A browser arriving at a page has *some* referer; its own origin is
+			// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
+			referer: referer || originOf(target),
+		}),
+		conditional: options.conditional,
+		timeoutMs,
+		deadline,
+		referer: options.referer,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	if (result.notModified) {
+		return {
+			notModified: true,
+			status: 304,
+			finalUrl: result.finalUrl,
+			html: null,
+			profileId: profile.id,
+		}
+	}
+
+	const declaredCharset = charsetFromContentType(result.contentType)
+	const { text, charset } = decodeBody(result.buffer, declaredCharset)
+
+	return {
+		notModified: false,
+		status: result.status,
+		finalUrl: result.finalUrl,
+		html: text,
+		charset,
+		contentType: result.contentType,
+		bytes: result.buffer.length,
+		etag: result.response.headers.get('etag') || null,
+		lastModified: result.response.headers.get('last-modified') || null,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch a binary (thumbnail image).
+//
+HttpFetch.fetchBinary = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || config.imageTimeoutMs
+	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
+	const maxBytes = options.maxBytes || config.maxImageBytes
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const acceptMimes = (mime) => {
+		if (!mime.startsWith('image/')) { return false }
+		if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
+		return true
+	}
+
+	const result = await withRetries(() => requestOnce(url, {
+		maxBytes,
+		acceptMimes,
+		headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
+			url: target,
+			pageUrl: options.pageUrl,
+		}),
+		timeoutMs,
+		deadline,
+	}), { retries: options.retries != null ? options.retries : config.retries, deadline })
+
+	return {
+		status: result.status,
+		finalUrl: result.finalUrl,
+		contentType: result.contentType,
+		mime: result.mime,
+		buffer: result.buffer,
+		profileId: profile.id,
+	}
+}
+
+//
+// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
+//
+HttpFetch.fetchText = async (url, options = {}) => {
+	const timeoutMs = options.timeoutMs || 5000
+	const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
+	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))
+
+	const result = await requestOnce(url, {
+		maxBytes: options.maxBytes || config.maxRobotsBytes,
+		acceptMimes: null, // robots.txt is served as every type under the sun
+		headersFor: () => BrowserProfiles.robotsHeaders(profile),
+		timeoutMs,
+		deadline,
+	})
+
+	if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }
+
+	const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
+	return { status: result.status, text, finalUrl: result.finalUrl }
+}
+
+const hostnameOf = (url) => {
+	try { return new URL(typeof url === 'string' ? url : url.href).hostname }
+	catch (error) { return '' }
+}
+
+const originOf = (url) => {
+	try { return new URL(url).origin + '/' }
+	catch (error) { return undefined }
+}
+
+//
+// Deliberately NOT attempted here, so nobody wonders later:
+//   - Header ordering: fetch normalises it; controlling order needs raw sockets.
+//   - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
+//   - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
+//     rather than evasive.
+//