require('module-alias/register')

const HttpFetch = require('@helpers/HttpFetch')
const BrowserProfiles = require('@helpers/BrowserProfiles')
const FetchMock = require('../support/fetchMock')

const dns = require('dns').promises
let dnsSpy = null

beforeEach(() => {
	FetchMock.install()
	// Every hostname in these tests resolves to one public address, so the
	// safety gate passes and we are testing HttpFetch rather than DNS.
	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
	// No real wall-time in backoff.
	HttpFetch.__setSleep(() => Promise.resolve())
})

afterEach(() => {
	FetchMock.restore()
	dnsSpy.mockRestore()
	HttpFetch.__resetSleep()
})

describe('fetchDocument happy path', () => {

	test('returns decoded html, status and final url', async () => {
		FetchMock.route('https://example.com/', FetchMock.html('<title>Hello</title>'))

		const result = await HttpFetch.fetchDocument('https://example.com/')

		expect(result.status).toBe(200)
		expect(result.html).toContain('<title>Hello</title>')
		expect(result.finalUrl).toBe('https://example.com/')
		expect(result.notModified).toBe(false)
	})

	test('surfaces etag and last-modified for later conditional requests', async () => {
		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>', {
			headers: { etag: 'W/"abc"', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' },
		}))

		const result = await HttpFetch.fetchDocument('https://example.com/')

		expect(result.etag).toBe('W/"abc"')
		expect(result.lastModified).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
	})
})

describe('fetchDocument browser identity', () => {

	test('sends a complete, coherent header set and never claims to be Googlebot', async () => {
		FetchMock.route('https://example.com/', FetchMock.html('<p>x</p>'))

		await HttpFetch.fetchDocument('https://example.com/')
		const headers = FetchMock.calls()[0].headers

		expect(headers['User-Agent']).toMatch(/^Mozilla\/5\.0/)
		expect(headers['User-Agent']).not.toMatch(/Googlebot/i)
		expect(headers['Accept']).toContain('text/html')
		expect(headers['Accept-Language']).toBeTruthy()
		expect(headers['Accept-Encoding']).toContain('gzip')
	})

	test('gives the same host the same profile across calls', async () => {
		FetchMock.route(/example\.com/, () => FetchMock.html('<p>x</p>'))

		const first = await HttpFetch.fetchDocument('https://example.com/a')
		const second = await HttpFetch.fetchDocument('https://example.com/b')

		expect(first.profileId).toBe(second.profileId)
	})

	test('pairs sec-ch-ua only with Chromium profiles', () => {
		const chrome = BrowserProfiles.profileById('chrome-win')
		const firefox = BrowserProfiles.profileById('firefox-win')
		const safari = BrowserProfiles.profileById('safari-mac')

		const chromeHeaders = BrowserProfiles.documentHeaders(chrome, { url: 'https://a.com/' })
		const firefoxHeaders = BrowserProfiles.documentHeaders(firefox, { url: 'https://a.com/' })
		const safariHeaders = BrowserProfiles.documentHeaders(safari, { url: 'https://a.com/' })

		expect(chromeHeaders['sec-ch-ua']).toBeTruthy()
		expect(chromeHeaders['Sec-Fetch-Mode']).toBe('navigate')

		// Firefox sends Sec-Fetch-* but no client hints.
		expect(firefoxHeaders['sec-ch-ua']).toBeUndefined()
		expect(firefoxHeaders['Sec-Fetch-Mode']).toBe('navigate')

		// Safari sends neither.
		expect(safariHeaders['sec-ch-ua']).toBeUndefined()
		expect(safariHeaders['Sec-Fetch-Mode']).toBeUndefined()
	})

	test('never pairs a chrome UA with a firefox accept header', () => {
		BrowserProfiles.PROFILES.forEach(profile => {
			if (profile.family === 'chrome') {
				expect(profile.ua).toMatch(/Chrome\//)
				expect(profile.accept).toContain('application/signed-exchange')
			}
			if (profile.family === 'firefox') {
				expect(profile.ua).toMatch(/Firefox\//)
				expect(profile.accept).not.toContain('signed-exchange')
			}
		})
	})
})

describe('fetchDocument redirects', () => {

	test('follows a single hop and reports the final url', async () => {
		FetchMock.route('https://example.com/a', FetchMock.redirect('https://example.com/b', 301))
		FetchMock.route('https://example.com/b', () => FetchMock.html('<title>B</title>'))

		const result = await HttpFetch.fetchDocument('https://example.com/a')

		expect(result.finalUrl).toBe('https://example.com/b')
		expect(result.html).toContain('B')
	})

	test('resolves a relative Location header', async () => {
		FetchMock.route('https://example.com/a', FetchMock.redirect('/c', 302))
		FetchMock.route('https://example.com/c', () => FetchMock.html('<title>C</title>'))

		const result = await HttpFetch.fetchDocument('https://example.com/a')
		expect(result.finalUrl).toBe('https://example.com/c')
	})

	test('detects a redirect loop', async () => {
		FetchMock.route('https://example.com/a', () => FetchMock.redirect('https://example.com/b', 302))
		FetchMock.route('https://example.com/b', () => FetchMock.redirect('https://example.com/a', 302))

		await expect(HttpFetch.fetchDocument('https://example.com/a'))
			.rejects.toMatchObject({ kind: 'redirect_loop' })
	})

	test('gives up past the redirect cap', async () => {
		let n = 0
		FetchMock.route(/https:\/\/example\.com\/hop/, () => FetchMock.redirect(`https://example.com/hop${++n}`, 302))

		await expect(HttpFetch.fetchDocument('https://example.com/hop'))
			.rejects.toMatchObject({ kind: 'too_many_redirects' })
	})

	// The SSRF case that matters: the first URL is perfectly public.
	test('blocks a redirect pointing at loopback', async () => {
		FetchMock.route('https://example.com/a', FetchMock.redirect('http://127.0.0.1:3000/admin', 302))

		await expect(HttpFetch.fetchDocument('https://example.com/a'))
			.rejects.toMatchObject({ kind: 'blocked' })
	})

	test('blocks a redirect pointing at the cloud metadata endpoint', async () => {
		FetchMock.route('https://example.com/a', FetchMock.redirect('http://169.254.169.254/latest/meta-data/', 302))

		await expect(HttpFetch.fetchDocument('https://example.com/a'))
			.rejects.toMatchObject({ kind: 'blocked' })
	})

	test('refuses an https to http downgrade', async () => {
		FetchMock.route('https://example.com/a', FetchMock.redirect('http://example.com/a', 302))

		await expect(HttpFetch.fetchDocument('https://example.com/a'))
			.rejects.toMatchObject({ kind: 'blocked' })
	})
})

describe('fetchDocument retries', () => {

	test('retries a 500 and succeeds', async () => {
		FetchMock.routeSequence('https://example.com/', [
			FetchMock.status(500),
			FetchMock.html('<title>Recovered</title>'),
		])

		const result = await HttpFetch.fetchDocument('https://example.com/')

		expect(result.html).toContain('Recovered')
		expect(FetchMock.callCount()).toBe(2)
	})

	test('gives up after exhausting retries', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.status(503))

		await expect(HttpFetch.fetchDocument('https://example.com/'))
			.rejects.toMatchObject({ kind: 'http', status: 503 })

		// 1 initial attempt + config.retries
		expect(FetchMock.callCount()).toBe(3)
	})

	test('does NOT retry a 404 - it is an answer, not a failure', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.status(404))

		await expect(HttpFetch.fetchDocument('https://example.com/'))
			.rejects.toMatchObject({ kind: 'http', status: 404 })

		expect(FetchMock.callCount()).toBe(1)
	})

	test('does NOT retry a 403', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.status(403))

		await expect(HttpFetch.fetchDocument('https://example.com/')).rejects.toMatchObject({ status: 403 })
		expect(FetchMock.callCount()).toBe(1)
	})

	test('honours a short Retry-After on a 429', async () => {
		const waits = []
		HttpFetch.__setSleep((ms) => { waits.push(ms); return Promise.resolve() })

		FetchMock.routeSequence('https://example.com/', [
			FetchMock.status(429, { headers: { 'retry-after': '2' } }),
			FetchMock.html('<title>OK</title>'),
		])

		const result = await HttpFetch.fetchDocument('https://example.com/')

		expect(result.html).toContain('OK')
		expect(waits[0]).toBe(2000)
	})

	test('refuses to sit on an hour-long Retry-After', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.status(429, { headers: { 'retry-after': '3600' } }))

		await expect(HttpFetch.fetchDocument('https://example.com/'))
			.rejects.toMatchObject({ kind: 'http', status: 429 })

		// Surfaced for the cache to park the URL rather than retried.
		expect(FetchMock.callCount()).toBe(1)
	})

	test('retries a network error', async () => {
		FetchMock.routeSequence('https://example.com/', [
			FetchMock.networkError('ECONNRESET'),
			FetchMock.html('<title>Up</title>'),
		])

		const result = await HttpFetch.fetchDocument('https://example.com/')
		expect(result.html).toContain('Up')
	})
})

describe('fetchDocument limits and gates', () => {

	test('times out on a server that never responds', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.hang())

		await expect(HttpFetch.fetchDocument('https://example.com/', { timeoutMs: 60, retries: 0 }))
			.rejects.toMatchObject({ kind: 'timeout' })
	})

	test('aborts a body that exceeds the cap mid-stream', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.oversized())

		await expect(HttpFetch.fetchDocument('https://example.com/', { maxBytes: 128 * 1024, retries: 0 }))
			.rejects.toMatchObject({ kind: 'too_large' })
	})

	test('rejects an oversized declared content-length without reading the body', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.html('<p>small</p>', {
			headers: { 'content-length': String(50 * 1024 * 1024) },
		}))

		await expect(HttpFetch.fetchDocument('https://example.com/', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'too_large' })
	})

	test('refuses a non-html content type before parsing', async () => {
		FetchMock.route('https://example.com/f.pdf', () => FetchMock.buffer(Buffer.alloc(16), {
			contentType: 'application/pdf',
		}))

		await expect(HttpFetch.fetchDocument('https://example.com/f.pdf', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'bad_content_type' })
	})

	test('refuses a binary masquerading behind an image content type', async () => {
		FetchMock.route('https://example.com/x', () => FetchMock.buffer(Buffer.alloc(16), {
			contentType: 'image/png',
		}))

		await expect(HttpFetch.fetchDocument('https://example.com/x', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'bad_content_type' })
	})
})

describe('fetchDocument charset handling', () => {

	test('decodes windows-1252 declared in the content-type header', async () => {
		// 0x93/0x94 are curly quotes in cp1252 and invalid in utf-8.
		const body = Buffer.concat([
			Buffer.from('<title>', 'latin1'),
			Buffer.from([0x93]),
			Buffer.from('caf', 'latin1'),
			Buffer.from([0xe9, 0x94]),
			Buffer.from('</title>', 'latin1'),
		])
		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
			contentType: 'text/html; charset=windows-1252',
		}))

		const result = await HttpFetch.fetchDocument('https://example.com/')

		expect(result.html).toContain('“café”')
		expect(result.charset).toBe('windows-1252')
	})

	test('falls back to a meta charset when the header says nothing', async () => {
		const body = Buffer.concat([
			Buffer.from('<meta charset="windows-1252"><title>caf', 'latin1'),
			Buffer.from([0xe9]),
			Buffer.from('</title>', 'latin1'),
		])
		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, { contentType: 'text/html' }))

		const result = await HttpFetch.fetchDocument('https://example.com/')
		expect(result.html).toContain('café')
	})

	test('header charset wins over the meta tag', async () => {
		const body = Buffer.from('<meta charset="windows-1252"><title>café</title>', 'utf8')
		FetchMock.route('https://example.com/', () => FetchMock.buffer(body, {
			contentType: 'text/html; charset=utf-8',
		}))

		const result = await HttpFetch.fetchDocument('https://example.com/')
		expect(result.html).toContain('café')
	})

	test('survives a bogus charset label', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.buffer(Buffer.from('<p>ok</p>'), {
			contentType: 'text/html; charset=not-a-real-charset',
		}))

		const result = await HttpFetch.fetchDocument('https://example.com/')
		expect(result.html).toContain('ok')
	})
})

describe('fetchDocument conditional requests', () => {

	test('sends If-None-Match and If-Modified-Since when given', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.html('<p>x</p>'))

		await HttpFetch.fetchDocument('https://example.com/', {
			conditional: { etag: 'W/"v1"', lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT' },
		})

		const headers = FetchMock.calls()[0].headers
		expect(headers['If-None-Match']).toBe('W/"v1"')
		expect(headers['If-Modified-Since']).toBe('Wed, 01 Jan 2025 00:00:00 GMT')
	})

	test('reports a 304 as notModified with no body', async () => {
		FetchMock.route('https://example.com/', () => FetchMock.status(304))

		const result = await HttpFetch.fetchDocument('https://example.com/', {
			conditional: { etag: 'W/"v1"' },
		})

		expect(result.notModified).toBe(true)
		expect(result.status).toBe(304)
		expect(result.html).toBeNull()
	})
})

describe('fetchBinary', () => {

	const pngMagic = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])

	test('returns the buffer for a real image', async () => {
		FetchMock.route('https://example.com/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))

		const result = await HttpFetch.fetchBinary('https://example.com/i.png')

		expect(result.mime).toBe('image/png')
		expect(result.buffer.equals(pngMagic)).toBe(true)
	})

	// The old code wrote a 404 HTML error page to disk and handed it to gm.
	test('refuses an html error page served where an image was expected', async () => {
		FetchMock.route('https://example.com/i.png', () => FetchMock.html('<h1>404</h1>'))

		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'bad_content_type' })
	})

	test('propagates a 404 status rather than writing anything', async () => {
		FetchMock.route('https://example.com/i.png', () => FetchMock.status(404))

		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'http', status: 404 })
	})

	test('enforces the image size cap', async () => {
		FetchMock.route('https://example.com/i.png', () => FetchMock.oversized('image/png'))

		await expect(HttpFetch.fetchBinary('https://example.com/i.png', { maxBytes: 64 * 1024, retries: 0 }))
			.rejects.toMatchObject({ kind: 'too_large' })
	})

	test('rejects svg by default', async () => {
		FetchMock.route('https://example.com/i.svg', () => FetchMock.buffer(Buffer.from('<svg/>'), {
			contentType: 'image/svg+xml',
		}))

		await expect(HttpFetch.fetchBinary('https://example.com/i.svg', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'bad_content_type' })
	})

	test('sends image-appropriate fetch metadata and a page referer', async () => {
		FetchMock.route('https://cdn.example.net/i.png', () => FetchMock.buffer(pngMagic, { contentType: 'image/png' }))

		await HttpFetch.fetchBinary('https://cdn.example.net/i.png', { pageUrl: 'https://example.com/post' })

		const headers = FetchMock.calls()[0].headers
		expect(headers['Referer']).toBe('https://example.com/post')
		expect(headers['Accept']).toContain('image/')
		if (headers['Sec-Fetch-Dest']) {
			expect(headers['Sec-Fetch-Dest']).toBe('image')
			expect(headers['Sec-Fetch-Site']).toBe('cross-site')
		}
	})
})

describe('transient vs terminal safety failures', () => {

	// A private address is a permanent refusal; a DNS failure means the host
	// may just be down. Classifying both as "blocked" parked transient
	// failures for the full cache TTL.
	test('classifies a private address as blocked', async () => {
		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))

		await expect(HttpFetch.fetchDocument('http://10.0.0.1/', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'blocked', reason: 'private_ip' })
	})

	test('classifies a DNS failure as network, not blocked', async () => {
		dnsSpy.mockRejectedValue(Object.assign(new Error('nope'), { code: 'ENOTFOUND' }))
		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))

		await expect(HttpFetch.fetchDocument('https://gone.example/', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'network', reason: 'dns_failed' })
	})

	test('retries a DNS failure, since it may be transient', async () => {
		let attempts = 0
		dnsSpy.mockImplementation(async () => {
			attempts++
			if (attempts < 2) { throw Object.assign(new Error('nope'), { code: 'ENOTFOUND' }) }
			return [{ address: '93.184.216.34', family: 4 }]
		})
		FetchMock.route('https://flaky.example/', () => FetchMock.html('<title>Back up</title>'))

		const result = await HttpFetch.fetchDocument('https://flaky.example/')
		expect(result.html).toContain('Back up')
		expect(attempts).toBe(2)
	})

	test('does NOT retry a private address', async () => {
		FetchMock.route(/.*/, () => FetchMock.html('<p>x</p>'))

		await expect(HttpFetch.fetchDocument('http://127.0.0.1/')).rejects.toMatchObject({ kind: 'blocked' })
		expect(FetchMock.callCount()).toBe(0)
	})
})

describe('safety gate on the initial url', () => {

	test('refuses to open a connection to a private address at all', async () => {
		FetchMock.route(/.*/, () => FetchMock.html('<p>should never be reached</p>'))

		await expect(HttpFetch.fetchDocument('http://127.0.0.1:3000/', { retries: 0 }))
			.rejects.toMatchObject({ kind: 'blocked' })

		expect(FetchMock.callCount()).toBe(0)
	})
})