SolidScribe/server/tests/models/ScrapeQueue.test.js

require('module-alias/register')

const db = require('@config/database')
const User = require('@models/User')
const Attachment = require('@models/Attachment')
const ScrapeCache = require('@models/ScrapeCache')
const ScrapeQueue = require('@workers/ScrapeQueue')
const Schema = require('@helpers/Schema')
const Robots = require('@helpers/Robots')
const HttpFetch = require('@helpers/HttpFetch')
const UrlSafety = require('@helpers/UrlSafety')
const FetchMock = require('../support/fetchMock')

const dns = require('dns').promises

const testUserName = 'jestTestUserScrapeQueue'
const password = 'Beans19934!!!'

let userId = null
let noteId = null
let dnsSpy = null

const page = (title, description) => FetchMock.html(`
	<html><head>
		<title>${title}</title>
		<meta property="og:description" content="${description || 'a description'}">
	</head><body><p>body</p></body></html>
`)

// Clear cache rows for a URL so each test starts from a real miss.
const clearCache = (url) => db.promise()
	.query('DELETE FROM scrape_cache WHERE url_hash = ?', [UrlSafety.urlKey(url)])

const attachmentRow = (id) => db.promise()
	.query('SELECT * FROM attachment WHERE id = ?', [id])
	.then(([rows]) => rows[0] || null)

beforeAll(async () => {
	// The server normally does this at boot; tests have no boot phase.
	const verification = await Schema.verifyScrapeSchema()
	if (!verification.ok) {
		throw new Error(`dev scrape schema not applied - run ./applyScrapeSchema.sh (${verification.remedy})`)
	}

	try {
		const existing = await User.getByUserName(testUserName)
		if (existing && existing.id) { await User.deleteUser(existing.id, password) }
	} catch (error) { /* first run */ }

	const registered = await User.register(testUserName, password)
	userId = registered.userId

	// A real note, so the soft-delete path below sees an existing note and
	// hides the attachment rather than deleting it.
	const [note] = await db.promise().query(
		'INSERT INTO note (user_id, created, opened) VALUES (?, ?, ?)',
		[userId, Math.round(Date.now() / 1000), Math.round(Date.now() / 1000)]
	)
	noteId = note.insertId
})

beforeEach(() => {
	FetchMock.install()
	Schema.__setReady(true)
	Robots.__resetCache()
	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
	HttpFetch.__setSleep(() => Promise.resolve())

	// Exercise the real code path with politeness spacing and robots disabled.
	ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
})

afterEach(() => {
	FetchMock.restore()
	dnsSpy.mockRestore()
	HttpFetch.__resetSleep()
	ScrapeQueue.__reset()
})

afterAll(async () => {
	await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
	await db.promise().query('DELETE FROM note WHERE user_id = ?', [userId])
	db.end()
})

describe('fetch and persist', () => {

	test('writes parsed metadata onto the attachment row as plaintext', async () => {
		const url = 'https://example.com/post-one'
		await clearCache(url)
		FetchMock.route(url, () => page('Post One', 'the first post'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		const row = await attachmentRow(attachmentId)

		expect(row.text).toContain('Post One')
		expect(row.text).toContain('the first post')
		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
		// Must stay unencrypted - the FULLTEXT index depends on it.
		expect(row.text).not.toMatch(/^[0-9a-f]{64}/)
	})

	test('keeps the url the user wrote rather than the canonical url', async () => {
		const url = 'https://example.com/original'
		await clearCache(url)

		FetchMock.route(url, () => FetchMock.html(`
			<html><head><title>Canonical Test</title>
			<link rel="canonical" href="https://example.com/different-canonical">
			</head><body>x</body></html>
		`))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		const row = await attachmentRow(attachmentId)
		expect(row.url).toBe(url)

		// The canonical URL is recorded in the cache, not on the attachment.
		const cached = await ScrapeCache.get(url)
		expect(cached.canonical_url).toBe('https://example.com/different-canonical')
	})
})

describe('caching', () => {

	test('a second request for the same url makes zero network calls', async () => {
		const url = 'https://example.com/cached'
		await clearCache(url)
		FetchMock.route(url, () => page('Cached Page'))

		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })

		const callsAfterFirst = FetchMock.callCount()
		expect(callsAfterFirst).toBeGreaterThan(0)

		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })

		expect(FetchMock.callCount()).toBe(callsAfterFirst)

		// The cached row still populates the second attachment.
		const row = await attachmentRow(secondId)
		expect(row.text).toContain('Cached Page')
		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
	})

	test('two notes sharing a url coalesce into one fetch', async () => {
		const url = 'https://example.com/shared'
		await clearCache(url)
		FetchMock.route(url, () => page('Shared Page'))

		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)

		// Enqueued together, so they overlap in flight.
		await ScrapeQueue.enqueueUrls([
			{ userId, noteId: null, attachmentId: firstId, url },
			{ userId, noteId: null, attachmentId: secondId, url },
		])

		expect(FetchMock.callsTo(url)).toHaveLength(1)

		const rows = await Promise.all([attachmentRow(firstId), attachmentRow(secondId)])
		rows.forEach(row => {
			expect(row.text).toContain('Shared Page')
			expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
		})
	})

	test('a stale entry revalidates and a 304 reuses the stored text', async () => {
		const url = 'https://example.com/revalidate'
		await clearCache(url)

		FetchMock.routeSequence(url, [
			FetchMock.html('<html><head><title>Original Title</title></head><body>x</body></html>', {
				headers: { etag: 'W/"v1"' },
			}),
			FetchMock.status(304),
		])

		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })

		// Force staleness so the next pass revalidates instead of hitting cache.
		await db.promise().query('UPDATE scrape_cache SET expires_at = ? WHERE url_hash = ?',
			[Math.round(Date.now() / 1000) - 10, UrlSafety.urlKey(url)])

		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })

		// Conditional request was sent.
		const conditional = FetchMock.callsTo(url).find(call => call.headers['If-None-Match'])
		expect(conditional).toBeTruthy()

		const row = await attachmentRow(secondId)
		expect(row.text).toContain('Original Title')
	})
})

describe('failure handling', () => {

	test('a 500 leaves the row failed and never writes a sentinel string', async () => {
		const url = 'https://example.com/broken'
		await clearCache(url)
		FetchMock.route(url, () => FetchMock.status(500))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		const row = await attachmentRow(attachmentId)

		expect(row.scrape_state).toBe(ScrapeQueue.STATE.FAILED)
		// The old code stored these in attachment.text and fed them to FULLTEXT.
		expect(row.text).not.toContain('No site text')
		expect(row.text).not.toContain('Request Timeout')
	})

	test('a 404 removes the shell row rather than orphaning it', async () => {
		const url = 'https://example.com/gone'
		await clearCache(url)
		FetchMock.route(url, () => FetchMock.status(404))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		expect(await attachmentRow(attachmentId)).toBeNull()
	})

	test('a blocked url removes the shell row and makes no request', async () => {
		const url = 'http://127.0.0.1:9999/internal'
		FetchMock.route(/.*/, () => page('should never load'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		expect(await attachmentRow(attachmentId)).toBeNull()
		expect(FetchMock.callCount()).toBe(0)
	})

	test('one failing url does not strand the rest of a batch', async () => {
		const good = 'https://example.com/good-one'
		const bad = 'https://example.com/bad-one'
		await clearCache(good)
		await clearCache(bad)

		FetchMock.route(good, () => page('Good One'))
		FetchMock.route(bad, () => FetchMock.status(500))

		const goodId = await Attachment.createPendingUrlAttachment(userId, null, good)
		const badId = await Attachment.createPendingUrlAttachment(userId, null, bad)

		// The old counter-based implementation never settled in this case.
		const texts = await ScrapeQueue.enqueueUrls([
			{ userId, noteId: null, attachmentId: badId, url: bad },
			{ userId, noteId: null, attachmentId: goodId, url: good },
		])

		expect(texts).toHaveLength(2)
		expect((await attachmentRow(goodId)).text).toContain('Good One')
	})

	test('a failed url is not refetched while inside its backoff window', async () => {
		const url = 'https://example.com/flaky'
		await clearCache(url)
		FetchMock.route(url, () => FetchMock.status(500))

		const firstId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: firstId, url })

		const callsAfterFirst = FetchMock.callCount()

		const secondId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId: secondId, url })

		expect(FetchMock.callCount()).toBe(callsAfterFirst)
	})
})

describe('robots.txt', () => {

	test('refuses a disallowed url and records it', async () => {
		const url = 'https://example.com/no-bots/page'
		await clearCache(url)

		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 }) // robots ON

		FetchMock.route('https://example.com/robots.txt', () => new Response(
			'User-agent: *\nDisallow: /no-bots', { status: 200, headers: { 'content-type': 'text/plain' } }
		))
		FetchMock.route(url, () => page('Should Not Be Fetched'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		expect(FetchMock.callsTo(url)).toHaveLength(0)

		const cached = await ScrapeCache.get(url)
		expect(cached.fetch_state).toBe('blocked')
	})

	test('proceeds when robots.txt allows the path', async () => {
		const url = 'https://example.com/allowed/page'
		await clearCache(url)

		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0 })

		FetchMock.route('https://example.com/robots.txt', () => new Response(
			'User-agent: *\nDisallow: /private', { status: 200, headers: { 'content-type': 'text/plain' } }
		))
		FetchMock.route(url, () => page('Allowed Page'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId: null, attachmentId, url })

		expect((await attachmentRow(attachmentId)).text).toContain('Allowed Page')
	})
})

//
// Production ran for a while with the scrape columns missing, and every note
// save attempted an INSERT that could not succeed. These pin the degraded
// behaviour: do nothing, quietly, and leave everything else working.
//
describe('degrades cleanly when the scrape schema is absent', () => {

	beforeEach(() => {
		Schema.__setReady(false)
	})

	test('scanTextForWebsites creates no rows and makes no requests', async () => {
		FetchMock.route(/.*/, () => page('Should Not Be Fetched'))

		const [before] = await db.promise().query(
			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])

		const text = await Attachment.scanTextForWebsites(userId, null, 'see https://example.com/unavailable')

		const [after] = await db.promise().query(
			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])

		expect(after[0].n).toBe(before[0].n)
		expect(FetchMock.callCount()).toBe(0)
		expect(typeof text).toBe('string')
	})

	//
	// Removing a link from a note must still clean up its attachment. That path
	// touches none of the new columns, so it is unaffected by the outage.
	//
	// Note the existing semantics: scanTextForWebsites calls
	// Attachment.delete(..., urlDelete = true), which HARD deletes. The
	// visible = 0 soft-delete applies only when a user removes an attachment
	// directly while its note still exists (urlDelete = false).
	//
	test('still removes an attachment whose url left the note', async () => {
		Schema.__setReady(true)
		const url = 'https://example.com/will-be-removed'
		await clearCache(url)
		FetchMock.route(url, () => page('Removed Later'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, noteId, url)
		await ScrapeQueue.enqueueUrl({ userId, noteId, attachmentId, url })
		expect(await attachmentRow(attachmentId)).not.toBeNull()

		// Now the schema "disappears" and the note is saved without that link.
		Schema.__setReady(false)
		await Attachment.scanTextForWebsites(userId, noteId, 'no links here any more')

		// Attachment.delete is deliberately fire-and-forget in scanTextForWebsites,
		// so give it a tick to land.
		await new Promise(resolve => setTimeout(resolve, 150))

		expect(await attachmentRow(attachmentId)).toBeNull()
	})

	test('scrapeUrlsCreateAttachments resolves empty without touching the DB', async () => {
		FetchMock.route(/.*/, () => page('nope'))

		await expect(Attachment.scrapeUrlsCreateAttachments(userId, null, ['https://example.com/x']))
			.resolves.toBe('')
		expect(FetchMock.callCount()).toBe(0)
	})

	test('pushUrl keeps its contract and resolves truthy', async () => {
		FetchMock.route(/.*/, () => page('nope'))

		await expect(Attachment.pushUrl('any-key', 'https://example.com/x')).resolves.toBe(true)
		expect(FetchMock.callCount()).toBe(0)
	})

	test('recoverPending returns zero without querying', async () => {
		await expect(ScrapeQueue.recoverPending()).resolves.toEqual({ requeued: 0, retired: 0 })
	})

	test('enqueueUrl is a no-op backstop', async () => {
		FetchMock.route(/.*/, () => page('nope'))

		await expect(ScrapeQueue.enqueueUrl({
			userId, noteId: null, attachmentId: null, url: 'https://example.com/x',
		})).resolves.toBe('')
		expect(FetchMock.callCount()).toBe(0)
	})
})

describe('crash recovery', () => {

	test('re-enqueues rows left pending by a restart', async () => {
		const url = 'https://example.com/recovered'
		await clearCache(url)
		FetchMock.route(url, () => page('Recovered Page'))

		// Simulate a row that was enqueued but never processed.
		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)

		const recovery = await ScrapeQueue.recoverPending()
		expect(recovery.requeued).toBeGreaterThan(0)

		await ScrapeQueue.drain()

		const row = await attachmentRow(attachmentId)
		expect(row.text).toContain('Recovered Page')
		expect(row.scrape_state).toBe(ScrapeQueue.STATE.DONE)
	})

	test('resets rows stuck mid-fetch so they are retried', async () => {
		const url = 'https://example.com/stuck'
		await clearCache(url)
		FetchMock.route(url, () => page('Unstuck Page'))

		const attachmentId = await Attachment.createPendingUrlAttachment(userId, null, url)
		// Pretend the process died while this one was in flight.
		await db.promise().query('UPDATE attachment SET scrape_state = ? WHERE id = ?',
			[ScrapeQueue.STATE.FETCHING, attachmentId])

		await ScrapeQueue.recoverPending()
		await ScrapeQueue.drain()

		expect((await attachmentRow(attachmentId)).scrape_state).toBe(ScrapeQueue.STATE.DONE)
	})

	test('does not sweep up long-settled rows', async () => {
		// Rows default to DONE, which is what keeps the pre-existing ~1400
		// attachment rows out of the recovery queue on first boot.
		const [rows] = await db.promise().query(
			'SELECT COUNT(*) AS n FROM attachment WHERE scrape_state = ? AND user_id = ?',
			[ScrapeQueue.STATE.PENDING, userId]
		)
		const recovery = await ScrapeQueue.recoverPending()
		expect(recovery.requeued).toBeLessThanOrEqual(rows[0].n + 1)
	})
})