Scraper Update for testing

server/tests/models/Attachment.test.js

@@ -1,5 +1,11 @@
 const Attachment = require('../../models/Attachment')
 const User = require('../../models/User')
+const ScrapeQueue = require('@workers/ScrapeQueue')
+const Robots = require('@helpers/Robots')
+const FetchMock = require('../support/fetchMock')
+const dns = require('dns').promises
+
+let dnsSpy = null
 
 const testUserName = 'jestTestUserAttachment'
 const password = 'Beans19934!!!'
@@ -67,19 +73,122 @@ test('Test get Push Key Bookmarklet', () => {
 })
 
 
-test('Test Push URL', () => {
+//
+// These used to hit https://www.solidscribe.com over the live network, so the
+// suite needed internet access and the assertion depended on whatever that page
+// happened to serve. fetch is mocked against a fixture instead.
+//
+describe('Push URL', () => {
 
-	let url = 'https://www.solidscribe.com'
+	beforeEach(() => {
+		FetchMock.install()
+		Robots.__resetCache()
+		dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }])
+		// Real code path, no politeness waiting and no robots lookups.
+		ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false })
+	})
 
-	return Attachment.pushUrl(newPushKey, url)
-	.then(( results => {
+	afterEach(() => {
+		FetchMock.restore()
+		dnsSpy.mockRestore()
+		ScrapeQueue.__reset()
+	})
 
-		return Attachment.textSearch(newUserId, 'scribe')
+	test('scrapes a pushed url into a searchable attachment', () => {
 
-	}))
-	.then((results) => {
+		const url = 'https://www.solidscribe.com/'
 
-		expect(results.length == 1).toBe(true)
+		FetchMock.route(url, () => FetchMock.html(`
+			<html><head>
+				<title>Solid Scribe</title>
+				<meta property="og:description" content="An encrypted note taking application.">
+			</head><body><p>x</p></body></html>
+		`))
+
+		return Attachment.pushUrl(newPushKey, url)
+			.then(() => ScrapeQueue.drain())
+			.then(() => Attachment.textSearch(newUserId, 'scribe'))
+			.then(results => {
+				expect(results.length == 1).toBe(true)
+				expect(results[0].text).toContain('Solid Scribe')
+			})
+	})
+
+	test('stores the description so it is searchable too', () => {
+
+		const url = 'https://www.solidscribe.com/encrypted'
+
+		FetchMock.route(url, () => FetchMock.html(`
+			<html><head>
+				<title>Encryption Page</title>
+				<meta property="og:description" content="Notes are sealed with aes256gcm.">
+			</head><body><p>x</p></body></html>
+		`))
+
+		return Attachment.pushUrl(newPushKey, url)
+			.then(() => ScrapeQueue.drain())
+			.then(() => Attachment.textSearch(newUserId, 'aes256gcm'))
+			.then(results => {
+				expect(results.length).toBeGreaterThan(0)
+			})
+	})
+
+	// The pushmebaby query param was previously passed through with no
+	// validation whatsoever.
+	test('refuses a pushed url pointing at loopback and makes no request', () => {
+
+		FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
+
+		return Attachment.pushUrl(newPushKey, 'http://127.0.0.1:3000/admin')
+			.then(result => {
+				expect(result).toBe(true)
+				expect(FetchMock.callCount()).toBe(0)
+			})
+	})
+
+	test('refuses a pushed url pointing at the cloud metadata endpoint', () => {
+
+		FetchMock.route(/.*/, () => FetchMock.html('<title>should never load</title>'))
+
+		return Attachment.pushUrl(newPushKey, 'http://169.254.169.254/latest/meta-data/')
+			.then(() => {
+				expect(FetchMock.callCount()).toBe(0)
+			})
+	})
+
+	test('resolves truthy for an unknown pushkey without scraping', () => {
+
+		FetchMock.route(/.*/, () => FetchMock.html('<title>nope</title>'))
+
+		return Attachment.pushUrl('not-a-real-pushkey', 'https://example.com/')
+			.then(result => {
+				expect(result).toBe(true)
+				expect(FetchMock.callCount()).toBe(0)
+			})
+	})
+
+	// Previously this could hang the HTTP response indefinitely.
+	test('returns within the budget when the server never responds', () => {
+
+		const url = 'https://www.solidscribe.com/slow'
+		FetchMock.route(url, () => FetchMock.hang())
+
+		ScrapeQueue.configure({
+			perHostMinSpacingMs: 0,
+			perHostMaxSpacingMs: 0,
+			checkRobots: false,
+			pageTimeoutMs: 80,
+			totalBudgetMs: 300,
+			retries: 0,
+		})
+
+		const startedAt = Date.now()
+
+		return Attachment.pushUrl(newPushKey, url)
+			.then(result => {
+				expect(result).toBe(true)
+				expect(Date.now() - startedAt).toBeLessThan(5000)
+			})
 	})
 })