Adding scrape files for testing

server/helpers/Robots.js

@@ -0,0 +1,216 @@
+//
+// robots.txt fetching, parsing and caching.
+//
+// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
+// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
+// a 404 or a flaky server is not a refusal and these are single-page fetches of
+// URLs a user explicitly saved.
+//
+
+const config = require('@config/scrape')
+const HttpFetch = require('@helpers/HttpFetch')
+
+let Robots = module.exports = {}
+
+// origin -> { rules, expiresAt }
+const cache = new Map()
+
+Robots.__resetCache = () => cache.clear()
+Robots.__cacheSize = () => cache.size
+
+const ALLOW_ALL = { groups: [], allowAll: true }
+
+//
+// Parse robots.txt into user-agent groups.
+//
+// Consecutive User-agent lines share one rule block, which is the part naive
+// parsers get wrong.
+//
+Robots.parse = (text) => {
+	if (!text || typeof text !== 'string') { return ALLOW_ALL }
+
+	const groups = []
+	let current = null
+	let expectingAgents = false
+
+	for (const rawLine of text.split(/\r?\n/)) {
+		const line = rawLine.replace(/#.*$/, '').trim()
+		if (!line) { continue }
+
+		const separator = line.indexOf(':')
+		if (separator === -1) { continue }
+
+		const field = line.slice(0, separator).trim().toLowerCase()
+		const value = line.slice(separator + 1).trim()
+
+		if (field === 'user-agent') {
+			// A User-agent line directly after another starts a shared group.
+			if (!current || !expectingAgents) {
+				current = { agents: [], rules: [], crawlDelayMs: null }
+				groups.push(current)
+			}
+			current.agents.push(value.toLowerCase())
+			expectingAgents = true
+			continue
+		}
+
+		if (!current) { continue } // directive before any user-agent
+		expectingAgents = false
+
+		if (field === 'disallow' || field === 'allow') {
+			// "Disallow:" with an empty value means allow everything.
+			if (field === 'disallow' && value === '') { continue }
+			current.rules.push({ allow: field === 'allow', path: value })
+			continue
+		}
+
+		if (field === 'crawl-delay') {
+			const seconds = Number(value)
+			if (Number.isFinite(seconds) && seconds >= 0) {
+				current.crawlDelayMs = Math.round(seconds * 1000)
+			}
+		}
+	}
+
+	return { groups, allowAll: groups.length === 0 }
+}
+
+//
+// Pick the group whose agent token best matches us. A named match always beats
+// the "*" wildcard, regardless of file order.
+//
+const groupFor = (rules, userAgentToken) => {
+	if (!rules || rules.allowAll) { return null }
+
+	const token = userAgentToken.toLowerCase()
+	let best = null
+	let bestScore = -1
+
+	for (const group of rules.groups) {
+		for (const agent of group.agents) {
+			let score = -1
+			if (agent === '*') { score = 0 }
+			else if (token.includes(agent)) { score = agent.length }
+
+			if (score > bestScore) {
+				bestScore = score
+				best = group
+			}
+		}
+	}
+
+	return best
+}
+
+//
+// Match a robots path pattern, supporting the * and $ wildcards.
+//
+const matchesPattern = (pattern, path) => {
+	if (pattern === '') { return false }
+	if (!pattern.includes('*') && !pattern.includes('$')) {
+		return path.startsWith(pattern)
+	}
+
+	const anchored = pattern.endsWith('$')
+	const body = anchored ? pattern.slice(0, -1) : pattern
+
+	const expression = body
+		.split('*')
+		.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+		.join('.*')
+
+	return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
+}
+
+//
+// Longest matching rule wins; Allow beats Disallow at equal length.
+//
+Robots.isPathAllowed = (rules, path, userAgentToken) => {
+	const group = groupFor(rules, userAgentToken)
+	if (!group) { return true }
+
+	let decision = true
+	let decisionLength = -1
+
+	for (const rule of group.rules) {
+		if (!matchesPattern(rule.path, path)) { continue }
+
+		const length = rule.path.length
+		if (length > decisionLength || (length === decisionLength && rule.allow)) {
+			decision = rule.allow
+			decisionLength = length
+		}
+	}
+
+	return decision
+}
+
+const evict = () => {
+	if (cache.size <= config.robotsMaxOrigins) { return }
+	// Map preserves insertion order, so the oldest key is first.
+	const oldest = cache.keys().next().value
+	cache.delete(oldest)
+}
+
+const getRules = async (origin) => {
+	const cached = cache.get(origin)
+	if (cached && cached.expiresAt > Date.now()) { return cached.rules }
+
+	let rules = ALLOW_ALL
+	let ttl = config.robotsTtlMs
+
+	try {
+		const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
+			maxBytes: config.maxRobotsBytes,
+		})
+		rules = Robots.parse(result.text)
+	} catch (error) {
+		// No robots.txt, or we could not reach it. Permissive, but cached for a
+		// short while only so a transient outage does not grant a day's pass.
+		rules = ALLOW_ALL
+		ttl = (error && error.kind === 'http' && error.status === 404)
+			? config.robotsNegativeTtlMs
+			: config.robotsErrorTtlMs
+	}
+
+	cache.set(origin, { rules, expiresAt: Date.now() + ttl })
+	evict()
+
+	return rules
+}
+Robots.__getRules = getRules
+
+const originAndPath = (url) => {
+	const parsed = new URL(typeof url === 'string' ? url : url.href)
+	return { origin: parsed.origin, path: parsed.pathname + parsed.search }
+}
+
+//
+// Is this URL allowed by its host's robots.txt?
+//
+Robots.isAllowed = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return false }
+
+	const rules = await getRules(parts.origin)
+	return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
+}
+
+//
+// Crawl-delay for this host, or null. Capped: a site asking for an hour between
+// requests gets its job failed rather than parked in the queue forever.
+//
+Robots.crawlDelayMs = async (url, userAgentToken) => {
+	let parts = null
+	try { parts = originAndPath(url) } catch (error) { return null }
+
+	const rules = await getRules(parts.origin)
+	const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())
+
+	if (!group || group.crawlDelayMs == null) { return null }
+	return group.crawlDelayMs
+}
+
+Robots.exceedsCrawlDelayCap = (delayMs) => {
+	return delayMs != null && delayMs > config.maxCrawlDelayMs
+}