Adding scrape files for testing

server/tests/helpers/Robots.test.js

@@ -0,0 +1,173 @@
+require('module-alias/register')
+
+const Robots = require('@helpers/Robots')
+const config = require('@config/scrape')
+const FetchMock = require('../support/fetchMock')
+
+const dns = require('dns').promises
+let dnsSpy = null
+
+const UA = config.robotsUaToken
+
+beforeEach(() => {
+	FetchMock.install()
+	Robots.__resetCache()
+	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+})
+
+afterEach(() => {
+	FetchMock.restore()
+	dnsSpy.mockRestore()
+	Robots.__resetCache()
+})
+
+const serveRobots = (body, origin = 'https://example.com') => {
+	FetchMock.route(`${origin}/robots.txt`, () => new Response(body, {
+		status: 200,
+		headers: { 'content-type': 'text/plain' },
+	}))
+}
+
+describe('Robots.parse', () => {
+
+	test('groups consecutive user-agent lines into one rule block', () => {
+		const rules = Robots.parse([
+			'User-agent: alpha',
+			'User-agent: beta',
+			'Disallow: /private',
+		].join('\n'))
+
+		expect(rules.groups).toHaveLength(1)
+		expect(rules.groups[0].agents).toEqual(['alpha', 'beta'])
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/private' }])
+	})
+
+	test('ignores comments and blank lines', () => {
+		const rules = Robots.parse('# a comment\n\nUser-agent: *\nDisallow: /x # trailing\n')
+		expect(rules.groups[0].rules).toEqual([{ allow: false, path: '/x' }])
+	})
+
+	test('treats an empty Disallow as no restriction', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow:')
+		expect(rules.groups[0].rules).toHaveLength(0)
+	})
+
+	test('parses crawl-delay', () => {
+		const rules = Robots.parse('User-agent: *\nCrawl-delay: 2.5')
+		expect(rules.groups[0].crawlDelayMs).toBe(2500)
+	})
+
+	test('an empty file allows everything', () => {
+		expect(Robots.parse('').allowAll).toBe(true)
+		expect(Robots.parse(null).allowAll).toBe(true)
+	})
+})
+
+describe('Robots.isPathAllowed', () => {
+
+	test('respects a simple disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /private')
+		expect(Robots.isPathAllowed(rules, '/private/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/public/x', UA)).toBe(true)
+	})
+
+	test('longest match wins, so a nested Allow overrides a broad Disallow', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /a\nAllow: /a/b')
+		expect(Robots.isPathAllowed(rules, '/a/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/a/b/c', UA)).toBe(true)
+	})
+
+	test('Allow beats Disallow at equal specificity', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /p\nAllow: /p')
+		expect(Robots.isPathAllowed(rules, '/p', UA)).toBe(true)
+	})
+
+	test('supports the * wildcard', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /*.pdf')
+		expect(Robots.isPathAllowed(rules, '/docs/report.pdf', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/docs/report.html', UA)).toBe(true)
+	})
+
+	test('supports the $ anchor', () => {
+		const rules = Robots.parse('User-agent: *\nDisallow: /x$')
+		expect(Robots.isPathAllowed(rules, '/x', UA)).toBe(false)
+		expect(Robots.isPathAllowed(rules, '/x/y', UA)).toBe(true)
+	})
+
+	test('a named group beats the wildcard group even when listed later', () => {
+		const rules = Robots.parse([
+			'User-agent: *',
+			'Disallow: /',
+			'',
+			`User-agent: ${UA}`,
+			'Disallow:',
+		].join('\n'))
+
+		expect(Robots.isPathAllowed(rules, '/anything', UA)).toBe(true)
+		expect(Robots.isPathAllowed(rules, '/anything', 'SomeOtherBot')).toBe(false)
+	})
+})
+
+describe('Robots.isAllowed over the network', () => {
+
+	test('refuses a disallowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/secret/page')).resolves.toBe(false)
+	})
+
+	test('permits an allowed path', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+		await expect(Robots.isAllowed('https://example.com/blog/post')).resolves.toBe(true)
+	})
+
+	// Fail-open: a missing robots.txt is not a refusal.
+	test('allows everything when robots.txt 404s', async () => {
+		FetchMock.route('https://example.com/robots.txt', () => FetchMock.status(404))
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('allows everything when robots.txt cannot be reached', async () => {
+		FetchMock.route('https://example.com/robots.txt', FetchMock.networkError())
+		await expect(Robots.isAllowed('https://example.com/anything')).resolves.toBe(true)
+	})
+
+	test('caches per origin - two queries, one network call', async () => {
+		serveRobots('User-agent: *\nDisallow: /secret')
+
+		await Robots.isAllowed('https://example.com/a')
+		await Robots.isAllowed('https://example.com/secret/b')
+
+		expect(FetchMock.callsTo('https://example.com/robots.txt')).toHaveLength(1)
+	})
+
+	test('keeps origins separate', async () => {
+		serveRobots('User-agent: *\nDisallow: /', 'https://blocked.com')
+		serveRobots('User-agent: *\nDisallow:', 'https://open.com')
+
+		await expect(Robots.isAllowed('https://blocked.com/x')).resolves.toBe(false)
+		await expect(Robots.isAllowed('https://open.com/x')).resolves.toBe(true)
+	})
+
+	test('returns false for an unparseable url rather than throwing', async () => {
+		await expect(Robots.isAllowed('not a url')).resolves.toBe(false)
+	})
+})
+
+describe('Robots.crawlDelayMs', () => {
+
+	test('reports the declared delay', async () => {
+		serveRobots('User-agent: *\nCrawl-delay: 5')
+		await expect(Robots.crawlDelayMs('https://example.com/x')).resolves.toBe(5000)
+	})
+
+	test('is null when none is declared', async () => {
+		serveRobots('User-agent: *\nDisallow: /x')
+		await expect(Robots.crawlDelayMs('https://example.com/y')).resolves.toBeNull()
+	})
+
+	test('flags a delay beyond the cap instead of parking the job forever', async () => {
+		expect(Robots.exceedsCrawlDelayCap(3600 * 1000)).toBe(true)
+		expect(Robots.exceedsCrawlDelayCap(2000)).toBe(false)
+		expect(Robots.exceedsCrawlDelayCap(null)).toBe(false)
+	})
+})