SolidScribe/server/tests/helpers/Robots.test.js

require('module-alias/register')

const Robots = require('@helpers/Robots')
const config = require('@config/scrape')
const FetchMock = require('../support/fetchMock')

const dns = require('dns').promises
let dnsSpy = null

const UA = config.robotsUaToken

beforeEach(() => {
	FetchMock.install()
	Robots.__resetCache()
	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
})

afterEach(() => {
	FetchMock.restore()
	dnsSpy.mockRestore()
	Robots.__resetCache()
})

const serveRobots = (body, origin = 'https://example.com') => {
	FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
		status: 200,
		headers: { 'content-type': 'text/plain' },
	}))
}

describe('Robots.parse', () => {

	test('groups consecutive user-agent lines into one rule block', () => {
		const rules = Robots.parse([
			'User-agent: alpha',
			'User-agent: beta',
			'Disallow: /private',
		].join('\n'))

		expect(rules.groups).toHaveLength(1)
		expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
	})

	test('ignores comments and blank lines', () => {
		const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
	})

	test('treats an empty Disallow as no restriction', () => {
		const rules = Robots.parse('User-agent: *\nDisallow:')
		expect(rules.groups[0].rules).toHaveLength(0)
	})

	test('parses crawl-delay', () => {
		const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
		expect(rules.groups[0].crawlDelayMs).toBe(2500)
	})

	test('an empty file allows everything', () => {
		expect(Robots.parse('').allowAll).toBe(true)
		expect(Robots.parse(null).allowAll).toBe(true)
	})
})

describe('Robots.isPathAllowed', () => {

	test('respects a simple disallow', () => {
		const rules = Robots.parse('User-agent: *\nDisallow: /private')
		expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
		expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
	})

	test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
		const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
		expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
		expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
	})

	test('Allow beats Disallow at equal specificity', () => {
		const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
		expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
	})

	test('supports the * wildcard', () => {
		const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
		expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
		expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
	})

	test('supports the $ anchor', () => {
		const rules = Robots.parse('User-agent: *\nDisallow: /x$')
		expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
		expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
	})

	test('a named group beats the wildcard group even when listed later', () => {
		const rules = Robots.parse([
			'User-agent: *',
			'Disallow: /',
			'',
			`User-agent: ${UA}`,
			'Disallow:',
		].join('\n'))

		expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
		expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
	})
})

describe('Robots.isAllowed over the network', () => {

	test('refuses a disallowed path', async () => {
		serveRobots('User-agent: *\nDisallow: /secret')
		await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
	})

	test('permits an allowed path', async () => {
		serveRobots('User-agent: *\nDisallow: /secret')
		await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
	})

	// Fail-open: a missing robots.txt is not a refusal.
	test('allows everything when robots.txt 404s', async () => {
		FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
	})

	test('allows everything when robots.txt cannot be reached', async () => {
		FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
	})

	test('caches per origin - two queries, one network call', async () => {
		serveRobots('User-agent: *\nDisallow: /secret')

		await Robots.isAllowed('https://example.com/a')
		await Robots.isAllowed('https://example.com/secret/b')

		expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
	})

	test('keeps origins separate', async () => {
		serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
		serveRobots('User-agent: *\nDisallow:', 'https://open.com')

		await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
		await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
	})

	test('returns false for an unparseable url rather than throwing', async () => {
		await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
	})
})

describe('Robots.crawlDelayMs', () => {

	test('reports the declared delay', async () => {
		serveRobots('User-agent: *\nCrawl-delay: 5')
		await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
	})

	test('is null when none is declared', async () => {
		serveRobots('User-agent: *\nDisallow: /x')
		await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
	})

	test('flags a delay beyond the cap instead of parking the job forever', async () => {
		expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
		expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
		expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
	})
})