SolidScribe/server/helpers/Robots.js

//
// robots.txt fetching, parsing and caching.
//
// Posture: polite, not evasive. An explicit Disallow is respected (fail-closed);
// a robots.txt we cannot retrieve is treated as permissive (fail-open), because
// a 404 or a flaky server is not a refusal and these are single-page fetches of
// URLs a user explicitly saved.
//

const config = require('@config/scrape')
const HttpFetch = require('@helpers/HttpFetch')

let Robots = module.exports = {}

// origin -> { rules, expiresAt }
const cache = new Map()

Robots.__resetCache = () => cache.clear()
Robots.__cacheSize = () => cache.size

const ALLOW_ALL = { groups: [], allowAll: true }

//
// Parse robots.txt into user-agent groups.
//
// Consecutive User-agent lines share one rule block, which is the part naive
// parsers get wrong.
//
Robots.parse = (text) => {
	if (!text || typeof text !== 'string') { return ALLOW_ALL }

	const groups = []
	let current = null
	let expectingAgents = false

	for (const rawLine of text.split(/\r?\n/)) {
		const line = rawLine.replace(/#.*$/, '').trim()
		if (!line) { continue }

		const separator = line.indexOf(':')
		if (separator === -1) { continue }

		const field = line.slice(0, separator).trim().toLowerCase()
		const value = line.slice(separator + 1).trim()

		if (field === 'user-agent') {
			// A User-agent line directly after another starts a shared group.
			if (!current || !expectingAgents) {
				current = { agents: [], rules: [], crawlDelayMs: null }
				groups.push(current)
			}
			current.agents.push(value.toLowerCase())
			expectingAgents = true
			continue
		}

		if (!current) { continue } // directive before any user-agent
		expectingAgents = false

		if (field === 'disallow' || field === 'allow') {
			// "Disallow:" with an empty value means allow everything.
			if (field === 'disallow' && value === '') { continue }
			current.rules.push({ allow: field === 'allow', path: value })
			continue
		}

		if (field === 'crawl-delay') {
			const seconds = Number(value)
			if (Number.isFinite(seconds) && seconds >= 0) {
				current.crawlDelayMs = Math.round(seconds * 1000)
			}
		}
	}

	return { groups, allowAll: groups.length === 0 }
}

//
// Pick the group whose agent token best matches us. A named match always beats
// the "*" wildcard, regardless of file order.
//
const groupFor = (rules, userAgentToken) => {
	if (!rules || rules.allowAll) { return null }

	const token = userAgentToken.toLowerCase()
	let best = null
	let bestScore = -1

	for (const group of rules.groups) {
		for (const agent of group.agents) {
			let score = -1
			if (agent === '*') { score = 0 }
			else if (token.includes(agent)) { score = agent.length }

			if (score > bestScore) {
				bestScore = score
				best = group
			}
		}
	}

	return best
}

//
// Match a robots path pattern, supporting the * and $ wildcards.
//
const matchesPattern = (pattern, path) => {
	if (pattern === '') { return false }
	if (!pattern.includes('*') && !pattern.includes('$')) {
		return path.startsWith(pattern)
	}

	const anchored = pattern.endsWith('$')
	const body = anchored ? pattern.slice(0, -1) : pattern

	const expression = body
		.split('*')
		.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
		.join('.*')

	return new RegExp('^' + expression + (anchored ? '$' : '')).test(path)
}

//
// Longest matching rule wins; Allow beats Disallow at equal length.
//
Robots.isPathAllowed = (rules, path, userAgentToken) => {
	const group = groupFor(rules, userAgentToken)
	if (!group) { return true }

	let decision = true
	let decisionLength = -1

	for (const rule of group.rules) {
		if (!matchesPattern(rule.path, path)) { continue }

		const length = rule.path.length
		if (length > decisionLength || (length === decisionLength && rule.allow)) {
			decision = rule.allow
			decisionLength = length
		}
	}

	return decision
}

const evict = () => {
	if (cache.size <= config.robotsMaxOrigins) { return }
	// Map preserves insertion order, so the oldest key is first.
	const oldest = cache.keys().next().value
	cache.delete(oldest)
}

const getRules = async (origin) => {
	const cached = cache.get(origin)
	if (cached && cached.expiresAt > Date.now()) { return cached.rules }

	let rules = ALLOW_ALL
	let ttl = config.robotsTtlMs

	try {
		const result = await HttpFetch.fetchText(`${origin}/robots.txt`, {
			maxBytes: config.maxRobotsBytes,
		})
		rules = Robots.parse(result.text)
	} catch (error) {
		// No robots.txt, or we could not reach it. Permissive, but cached for a
		// short while only so a transient outage does not grant a day's pass.
		rules = ALLOW_ALL
		ttl = (error && error.kind === 'http' && error.status === 404)
			? config.robotsNegativeTtlMs
			: config.robotsErrorTtlMs
	}

	cache.set(origin, { rules, expiresAt: Date.now() + ttl })
	evict()

	return rules
}
Robots.__getRules = getRules

const originAndPath = (url) => {
	const parsed = new URL(typeof url === 'string' ? url : url.href)
	return { origin: parsed.origin, path: parsed.pathname + parsed.search }
}

//
// Is this URL allowed by its host's robots.txt?
//
Robots.isAllowed = async (url, userAgentToken) => {
	let parts = null
	try { parts = originAndPath(url) } catch (error) { return false }

	const rules = await getRules(parts.origin)
	return Robots.isPathAllowed(rules, parts.path, userAgentToken || config.robotsUaToken)
}

//
// Crawl-delay for this host, or null. Capped: a site asking for an hour between
// requests gets its job failed rather than parked in the queue forever.
//
Robots.crawlDelayMs = async (url, userAgentToken) => {
	let parts = null
	try { parts = originAndPath(url) } catch (error) { return null }

	const rules = await getRules(parts.origin)
	const group = groupFor(rules, (userAgentToken || config.robotsUaToken).toLowerCase())

	if (!group || group.crawlDelayMs == null) { return null }
	return group.crawlDelayMs
}

Robots.exceedsCrawlDelayCap = (delayMs) => {
	return delayMs != null && delayMs > config.maxCrawlDelayMs
}