Deploy and schema updates for testing

server/tests/models/Attachment.test.js

@@ -1,6 +1,7 @@
 const Attachment = require('../../models/Attachment')
 const User = require('../../models/User')
 const ScrapeQueue = require('@workers/ScrapeQueue')
+const Schema = require('@helpers/Schema')
 const Robots = require('@helpers/Robots')
 const FetchMock = require('../support/fetchMock')
 const dns = require('dns').promises
@@ -82,6 +83,8 @@ describe('Push URL', () => {
 
 	beforeEach(() => {
 		FetchMock.install()
+		// The server marks this at boot; there is no boot phase in tests.
+		Schema.__setReady(true)
 		Robots.__resetCache()
 		dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
 		// Real code path, no politeness waiting and no robots lookups.
@@ -92,6 +95,7 @@ describe('Push URL', () => {
 		FetchMock.restore()
 		dnsSpy.mockRestore()
 		ScrapeQueue.__reset()
+		Schema.__reset()
 	})
 
 	test('scrapes a pushed url into a searchable attachment', () => {

server/tests/models/ScrapeQueue.test.js

@@ -17,6 +17,7 @@ const testUserName = 'jestTestUserScrapeQueue'
 const password = 'Beans19934!!!'
 
 let userId = null
+let noteId = null
 let dnsSpy = null
 
 const page = (title, description) => FetchMock.html(`
@@ -35,7 +36,11 @@ const attachmentRow = (id) => db.promise()
 	.then(([rows]) => rows[0] || null)
 
 beforeAll(async () => {
-	await Schema.ensureScrapeSchema()
+	// The server normally does this at boot; tests have no boot phase.
+	const verification = await Schema.verifyScrapeSchema()
+	if (!verification.ok) {
+		throw new Error(`dev scrape schema not applied - run ./applyScrapeSchema.sh (${verification.remedy})`)
+	}
 
 	try {
 		const existing = await User.getByUserName(testUserName)
@@ -44,10 +49,19 @@ beforeAll(async () => {
 
 	const registered = await User.register(testUserName, password)
 	userId = registered.userId
+
+	// A real note, so the soft-delete path below sees an existing note and
+	// hides the attachment rather than deleting it.
+	const [note] = await db.promise().query(
+		'INSERT INTO note (user_id, created, opened) VALUES (?, ?, ?)',
+		[userId, Math.round(Date.now() / 1000), Math.round(Date.now() / 1000)]
+	)
+	noteId = note.insertId
 })
 
 beforeEach(() => {
 	FetchMock.install()
+	Schema.__setReady(true)
 	Robots.__resetCache()
 	dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
 	HttpFetch.__setSleep(() => Promise.resolve())
@@ -65,6 +79,7 @@ afterEach(() => {
 
 afterAll(async () => {
 	await db.promise().query('DELETE FROM attachment WHERE user_id = ?', [userId])
+	await db.promise().query('DELETE FROM note WHERE user_id = ?', [userId])
 	db.end()
 })
 
@@ -305,6 +320,92 @@ describe('robots.txt', () => {
 	})
 })
 
+//
+// Production ran for a while with the scrape columns missing, and every note
+// save attempted an INSERT that could not succeed. These pin the degraded
+// behaviour: do nothing, quietly, and leave everything else working.
+//
+describe('degrades cleanly when the scrape schema is absent', () => {
+
+	beforeEach(() => {
+		Schema.__setReady(false)
+	})
+
+	test('scanTextForWebsites creates no rows and makes no requests', async () => {
+		FetchMock.route(/.*/, () => page('Should Not Be Fetched'))
+
+		const [before] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
+
+		const text = await Attachment.scanTextForWebsites(userId, null, 'see https://example.com/unavailable')
+
+		const [after] = await db.promise().query(
+			'SELECT COUNT(*) AS n FROM attachment WHERE user_id = ?', [userId])
+
+		expect(after[0].n).toBe(before[0].n)
+		expect(FetchMock.callCount()).toBe(0)
+		expect(typeof text).toBe('string')
+	})
+
+	//
+	// Removing a link from a note must still clean up its attachment. That path
+	// touches none of the new columns, so it is unaffected by the outage.
+	//
+	// Note the existing semantics: scanTextForWebsites calls
+	// Attachment.delete(..., urlDelete = true), which HARD deletes. The
+	// visible = 0 soft-delete applies only when a user removes an attachment
+	// directly while its note still exists (urlDelete = false).
+	//
+	test('still removes an attachment whose url left the note', async () => {
+		Schema.__setReady(true)
+		const url = 'https://example.com/will-be-removed'
+		await clearCache(url)
+		FetchMock.route(url, () => page('Removed Later'))
+
+		const attachmentId = await Attachment.createPendingUrlAttachment(userId, noteId, url)
+		await ScrapeQueue.enqueueUrl({ userId, noteId, attachmentId, url })
+		expect(await attachmentRow(attachmentId)).not.toBeNull()
+
+		// Now the schema "disappears" and the note is saved without that link.
+		Schema.__setReady(false)
+		await Attachment.scanTextForWebsites(userId, noteId, 'no links here any more')
+
+		// Attachment.delete is deliberately fire-and-forget in scanTextForWebsites,
+		// so give it a tick to land.
+		await new Promise(resolve => setTimeout(resolve, 150))
+
+		expect(await attachmentRow(attachmentId)).toBeNull()
+	})
+
+	test('scrapeUrlsCreateAttachments resolves empty without touching the DB', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(Attachment.scrapeUrlsCreateAttachments(userId, null, ['https://example.com/x']))
+			.resolves.toBe('')
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('pushUrl keeps its contract and resolves truthy', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(Attachment.pushUrl('any-key', 'https://example.com/x')).resolves.toBe(true)
+		expect(FetchMock.callCount()).toBe(0)
+	})
+
+	test('recoverPending returns zero without querying', async () => {
+		await expect(ScrapeQueue.recoverPending()).resolves.toEqual({ requeued: 0, retired: 0 })
+	})
+
+	test('enqueueUrl is a no-op backstop', async () => {
+		FetchMock.route(/.*/, () => page('nope'))
+
+		await expect(ScrapeQueue.enqueueUrl({
+			userId, noteId: null, attachmentId: null, url: 'https://example.com/x',
+		})).resolves.toBe('')
+		expect(FetchMock.callCount()).toBe(0)
+	})
+})
+
 describe('crash recovery', () => {
 
 	test('re-enqueues rows left pending by a restart', async () => {