SolidScribe/server/helpers/HttpFetch.js

//
// The only module in the server that opens an outbound connection.
//
// Everything that made the old scraper unreliable is handled here, in one place:
// a single AbortController-based timeout (the old code raced request's socket
// timeout against a manual setTimeout that could fire against a null reference
// and crash the process), manual redirect following so the SSRF gate runs on
// every hop, hard size caps enforced while streaming, a Content-Type gate before
// any body is read, real charset decoding, and retries that only fire for
// transient failures.
//

const config = require('@config/scrape')
const UrlSafety = require('@helpers/UrlSafety')
const BrowserProfiles = require('@helpers/BrowserProfiles')

let HttpFetch = module.exports = {}

const HTML_TYPES = ['text/html', 'application/xhtml+xml', 'text/plain', 'application/xml', 'text/xml']

class FetchError extends Error {
	constructor(kind, message, extra = {}) {
		super(message)
		this.name = 'FetchError'
		this.kind = kind          // timeout|http|blocked|too_large|bad_content_type|redirect_loop|too_many_redirects|network|aborted
		this.status = extra.status || null
		this.retryAfterMs = extra.retryAfterMs || null
		this.url = extra.url || null
		this.reason = extra.reason || null
	}
}
HttpFetch.FetchError = FetchError

// Injectable so tests don't spend real wall-time in backoff. Far simpler than
// interleaving fake timers with promises.
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
HttpFetch.__setSleep = (fn) => { sleep = fn }
HttpFetch.__resetSleep = () => { sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) }

const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 507, 509, 521, 522, 523, 524])

const isRetryable = (error) => {
	if (!(error instanceof FetchError)) { return false }
	if (error.kind === 'network' || error.kind === 'timeout') { return true }
	if (error.kind === 'http') { return RETRYABLE_STATUS.has(error.status) }
	return false
}

// Retry-After is either delta-seconds or an HTTP-date.
const parseRetryAfter = (value) => {
	if (!value) { return null }

	const seconds = Number(value)
	if (Number.isFinite(seconds)) { return Math.max(0, seconds * 1000) }

	const when = Date.parse(value)
	if (Number.isFinite(when)) { return Math.max(0, when - Date.now()) }

	return null
}

const backoffFor = (attempt) => {
	const base = Math.min(config.backoffMaxMs, config.backoffBaseMs * Math.pow(2, attempt))
	return Math.round(base * (0.5 + Math.random())) // +/-50% jitter
}

const contentTypeOf = (response) => String(response.headers.get('content-type') || '').toLowerCase()
const mimeOf = (contentType) => contentType.split(';')[0].trim()

const charsetFromContentType = (contentType) => {
	const match = contentType.match(/charset=\s*"?([^";\s]+)"?/i)
	return match ? match[1].toLowerCase() : null
}

// Last resort: sniff a <meta charset> out of the head, read as latin1 so every
// byte maps to a character and the regex can't miss.
const charsetFromMeta = (buffer) => {
	const head = buffer.subarray(0, 2048).toString('latin1')

	const metaCharset = head.match(/<meta[^>]+charset\s*=\s*["']?\s*([a-z0-9_\-:]+)/i)
	if (metaCharset) { return metaCharset[1].toLowerCase() }

	const httpEquiv = head.match(/<meta[^>]+content\s*=\s*["'][^"']*charset=\s*([a-z0-9_\-:]+)/i)
	if (httpEquiv) { return httpEquiv[1].toLowerCase() }

	return null
}

const charsetFromBom = (buffer) => {
	if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) { return 'utf-8' }
	if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { return 'utf-16le' }
	if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { return 'utf-16be' }
	return null
}

const decodeBody = (buffer, declaredCharset) => {
	const label = declaredCharset || charsetFromBom(buffer) || charsetFromMeta(buffer) || 'utf-8'

	try {
		return { text: new TextDecoder(label, { fatal: false }).decode(buffer), charset: label }
	} catch (error) {
		// Unknown/bogus label - utf-8 with replacement chars beats throwing.
		return { text: new TextDecoder('utf-8', { fatal: false }).decode(buffer), charset: 'utf-8' }
	}
}

//
// Read the body with a running byte count, aborting mid-stream once the cap is
// passed. Content-Length is checked by the caller too, but it is optional and
// can lie, so this is the cap that actually holds.
//
const readCapped = async (response, maxBytes, controller, url) => {
	if (!response.body) { return Buffer.alloc(0) }

	const chunks = []
	let total = 0

	for await (const chunk of response.body) {
		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
		total += buffer.length

		if (total > maxBytes) {
			try { controller.abort() } catch (error) { /* already done */ }
			throw new FetchError('too_large', `Body exceeded ${maxBytes} bytes`, { url })
		}

		chunks.push(buffer)
	}

	return Buffer.concat(chunks, total)
}

//
// One HTTP request, following redirects by hand.
//
// Manual redirects are the whole point: fetch's own 'follow' would hop to
// 127.0.0.1 without ever re-consulting the safety gate.
//
const requestOnce = async (startUrl, options) => {
	const {
		maxBytes,
		acceptMimes,
		headersFor,
		conditional,
		timeoutMs,
		deadline,
		referer,
	} = options

	let currentUrl = typeof startUrl === 'string' ? startUrl : startUrl.href
	const visited = new Set()

	for (let hop = 0; hop <= config.maxRedirects; hop++) {
		if (visited.has(currentUrl)) {
			throw new FetchError('redirect_loop', `Redirect loop at ${currentUrl}`, { url: currentUrl })
		}
		visited.add(currentUrl)

		// Re-validate on every hop. This is the check the old code never had.
		try {
			await UrlSafety.assertFetchable(currentUrl)
		} catch (error) {
			// A DNS failure is transient - the host may simply be down right
			// now - so it must not be classified alongside "this address is
			// private", which is a permanent refusal. Collapsing the two meant
			// a briefly-unresolvable domain got parked for the full cache TTL
			// and had its attachment row deleted.
			const kind = error.reason === 'dns_failed' ? 'network' : 'blocked'
			throw new FetchError(kind, error.message, { url: currentUrl, reason: error.reason })
		}

		const remaining = deadline - Date.now()
		if (remaining <= 0) {
			throw new FetchError('timeout', 'Total budget exhausted', { url: currentUrl })
		}

		const controller = new AbortController()
		const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remaining))

		let response = null
		try {
			const headers = headersFor(currentUrl, referer)

			if (conditional && conditional.etag) { headers['If-None-Match'] = conditional.etag }
			if (conditional && conditional.lastModified) { headers['If-Modified-Since'] = conditional.lastModified }

			response = await fetch(currentUrl, {
				method: 'GET',
				redirect: 'manual',
				signal: controller.signal,
				headers,
			})
		} catch (error) {
			clearTimeout(timer)

			if (controller.signal.aborted) {
				throw new FetchError('timeout', `Timed out after ${timeoutMs}ms`, { url: currentUrl })
			}
			throw new FetchError('network', error.message || 'Network failure', { url: currentUrl })
		}

		// Redirect?
		if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
			clearTimeout(timer)
			try { if (response.body) { await response.body.cancel() } } catch (error) { /* nothing to drain */ }

			const location = response.headers.get('location')
			let next = null
			try {
				next = new URL(location, currentUrl)
			} catch (error) {
				throw new FetchError('network', `Unparseable redirect target ${location}`, { url: currentUrl })
			}

			// Never follow https -> http. A downgrade mid-chain is either an
			// attack or a misconfiguration, and is not worth the preview.
			if (new URL(currentUrl).protocol === 'https:' && next.protocol === 'http:') {
				throw new FetchError('blocked', `Refusing https->http downgrade to ${next.href}`, { url: currentUrl })
			}

			currentUrl = next.href
			continue
		}

		clearTimeout(timer)

		if (response.status === 304) {
			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
			return { notModified: true, status: 304, finalUrl: currentUrl, response }
		}

		if (!response.ok) {
			try { if (response.body) { await response.body.cancel() } } catch (error) { /* no body */ }
			throw new FetchError('http', `HTTP ${response.status} for ${currentUrl}`, {
				status: response.status,
				retryAfterMs: parseRetryAfter(response.headers.get('retry-after')),
				url: currentUrl,
			})
		}

		const contentType = contentTypeOf(response)
		const mime = mimeOf(contentType)

		// Gate on type BEFORE reading a byte. This is what stops a 100MB binary
		// being handed to cheerio.
		if (acceptMimes && !acceptMimes(mime)) {
			try { controller.abort() } catch (error) { /* ignore */ }
			throw new FetchError('bad_content_type', `Refusing content-type ${mime || '(none)'}`, { url: currentUrl })
		}

		const declaredLength = Number(response.headers.get('content-length'))
		if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
			try { controller.abort() } catch (error) { /* ignore */ }
			throw new FetchError('too_large', `Declared length ${declaredLength} over cap`, { url: currentUrl })
		}

		const buffer = await readCapped(response, maxBytes, controller, currentUrl)

		return {
			notModified: false,
			status: response.status,
			finalUrl: currentUrl,
			contentType,
			mime,
			buffer,
			response,
		}
	}

	throw new FetchError('too_many_redirects', `More than ${config.maxRedirects} redirects`, { url: currentUrl })
}

//
// Retry wrapper. Only transient classes are retried - a 404 is an answer.
//
const withRetries = async (attemptFn, { retries, deadline }) => {
	let lastError = null

	for (let attempt = 0; attempt <= retries; attempt++) {
		try {
			return await attemptFn()
		} catch (error) {
			lastError = error

			if (!isRetryable(error) || attempt === retries) { throw error }

			// A server asking us to come back in an hour is not something to
			// sit on - surface it so the cache can park the URL instead.
			if (error.retryAfterMs && error.retryAfterMs > config.maxRetryAfterMs) { throw error }

			const wait = error.retryAfterMs != null
				? Math.min(error.retryAfterMs, config.maxRetryAfterMs)
				: backoffFor(attempt)

			if (Date.now() + wait >= deadline) { throw error }

			await sleep(wait)
		}
	}

	throw lastError
}

//
// Fetch an HTML document.
//
HttpFetch.fetchDocument = async (url, options = {}) => {
	const timeoutMs = options.timeoutMs || config.pageTimeoutMs
	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
	const maxBytes = options.maxBytes || config.maxHtmlBytes
	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))

	const result = await withRetries(() => requestOnce(url, {
		maxBytes,
		acceptMimes: (mime) => HTML_TYPES.includes(mime),
		headersFor: (target, referer) => BrowserProfiles.documentHeaders(profile, {
			url: target,
			// A browser arriving at a page has *some* referer; its own origin is
			// the least surprising one and pairs with Sec-Fetch-Site: same-origin.
			referer: referer || originOf(target),
		}),
		conditional: options.conditional,
		timeoutMs,
		deadline,
		referer: options.referer,
	}), { retries: options.retries != null ? options.retries : config.retries, deadline })

	if (result.notModified) {
		return {
			notModified: true,
			status: 304,
			finalUrl: result.finalUrl,
			html: null,
			profileId: profile.id,
		}
	}

	const declaredCharset = charsetFromContentType(result.contentType)
	const { text, charset } = decodeBody(result.buffer, declaredCharset)

	return {
		notModified: false,
		status: result.status,
		finalUrl: result.finalUrl,
		html: text,
		charset,
		contentType: result.contentType,
		bytes: result.buffer.length,
		etag: result.response.headers.get('etag') || null,
		lastModified: result.response.headers.get('last-modified') || null,
		profileId: profile.id,
	}
}

//
// Fetch a binary (thumbnail image).
//
HttpFetch.fetchBinary = async (url, options = {}) => {
	const timeoutMs = options.timeoutMs || config.imageTimeoutMs
	const deadline = Date.now() + (options.budgetMs || config.totalBudgetMs)
	const maxBytes = options.maxBytes || config.maxImageBytes
	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))

	const acceptMimes = (mime) => {
		if (!mime.startsWith('image/')) { return false }
		if (mime === 'image/svg+xml' && !config.allowSvgThumbnails) { return false }
		return true
	}

	const result = await withRetries(() => requestOnce(url, {
		maxBytes,
		acceptMimes,
		headersFor: (target) => BrowserProfiles.imageHeaders(profile, {
			url: target,
			pageUrl: options.pageUrl,
		}),
		timeoutMs,
		deadline,
	}), { retries: options.retries != null ? options.retries : config.retries, deadline })

	return {
		status: result.status,
		finalUrl: result.finalUrl,
		contentType: result.contentType,
		mime: result.mime,
		buffer: result.buffer,
		profileId: profile.id,
	}
}

//
// Fetch robots.txt. Short leash, no retries - fail-open is handled by the caller.
//
HttpFetch.fetchText = async (url, options = {}) => {
	const timeoutMs = options.timeoutMs || 5000
	const deadline = Date.now() + (options.budgetMs || timeoutMs + 1000)
	const profile = options.profile || BrowserProfiles.profileForHost(hostnameOf(url))

	const result = await requestOnce(url, {
		maxBytes: options.maxBytes || config.maxRobotsBytes,
		acceptMimes: null, // robots.txt is served as every type under the sun
		headersFor: () => BrowserProfiles.robotsHeaders(profile),
		timeoutMs,
		deadline,
	})

	if (result.notModified) { return { status: 304, text: null, finalUrl: result.finalUrl } }

	const { text } = decodeBody(result.buffer, charsetFromContentType(result.contentType))
	return { status: result.status, text, finalUrl: result.finalUrl }
}

const hostnameOf = (url) => {
	try { return new URL(typeof url === 'string' ? url : url.href).hostname }
	catch (error) { return '' }
}

const originOf = (url) => {
	try { return new URL(url).origin + '/' }
	catch (error) { return undefined }
}

//
// Deliberately NOT attempted here, so nobody wonders later:
//   - Header ordering: fetch normalises it; controlling order needs raw sockets.
//   - HTTP/2: undici's fetch is h1 unless an Agent with allowH2 is supplied.
//   - TLS/JA3 fingerprinting and proxy rotation: ruled out, we are being polite
//     rather than evasive.
//