const Attachment = require('../../models/Attachment') const User = require('../../models/User') const ScrapeQueue = require('@workers/ScrapeQueue') const Schema = require('@helpers/Schema') const Robots = require('@helpers/Robots') const FetchMock = require('../support/fetchMock') const dns = require('dns').promises let dnsSpy = null const testUserName = 'jestTestUserAttachment' const password = 'Beans19934!!!' let newUserId = null let masterKey = null let newPushKey = null beforeAll(() => { // Find and Delete Previous Test user, log in, get key return User.getByUserName(testUserName) .then((user) => { return User.deleteUser(user?.id, password) }) .then((results) => { return User.register(testUserName, password) }) .then(({ token, userId }) => { newUserId = userId return User.getMasterKey(userId, password) }) .then((newMasterKey) => { masterKey = newMasterKey return true }) .catch(((error) => { console.log(error) })) }) test('Test Generate Push Key', () => { return Attachment.generatePushKey(newUserId) .then( (pushKey) => { newPushKey = pushKey return Attachment.generatePushKey(newUserId) }) .then( (pushKey) => { // expect a long, defined pushkey expect(pushKey).toBeDefined() expect(pushKey?.length).toBeGreaterThan(20) expect(pushKey).toMatch(newPushKey) }) }) test('Test get Push Key Bookmarklet', () => { return Attachment.getPushkeyBookmarklet(newUserId) .then(( bookmarklet => { // Expect a bookmarklet containting URL encoded pushkey from above const keyCheck = bookmarklet.includes(encodeURIComponent(newPushKey)) expect(bookmarklet).toBeDefined() expect(keyCheck).toBe(true) })) }) // // These used to hit https://www.solidscribe.com over the live network, so the // suite needed internet access and the assertion depended on whatever that page // happened to serve. fetch is mocked against a fixture instead. // describe('Push URL', () => { beforeEach(() => { FetchMock.install() // The server marks this at boot; there is no boot phase in tests. Schema.__setReady(true) Robots.__resetCache() dnsSpy = jest.spyOn(dns, 'lookup').mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) // Real code path, no politeness waiting and no robots lookups. ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false }) }) afterEach(() => { FetchMock.restore() dnsSpy.mockRestore() ScrapeQueue.__reset() Schema.__reset() }) test('scrapes a pushed url into a searchable attachment', () => { const url = 'https://www.solidscribe.com/' FetchMock.route(url, () => FetchMock.html(` Solid Scribe

x

`)) return Attachment.pushUrl(newPushKey, url) .then(() => ScrapeQueue.drain()) .then(() => Attachment.textSearch(newUserId, 'scribe')) .then(results => { expect(results.length == 1).toBe(true) expect(results[0].text).toContain('Solid Scribe') }) }) test('stores the description so it is searchable too', () => { const url = 'https://www.solidscribe.com/encrypted' FetchMock.route(url, () => FetchMock.html(` Encryption Page

x

`)) return Attachment.pushUrl(newPushKey, url) .then(() => ScrapeQueue.drain()) .then(() => Attachment.textSearch(newUserId, 'aes256gcm')) .then(results => { expect(results.length).toBeGreaterThan(0) }) }) // The pushmebaby query param was previously passed through with no // validation whatsoever. test('refuses a pushed url pointing at loopback and makes no request', () => { FetchMock.route(/.*/, () => FetchMock.html('should never load')) return Attachment.pushUrl(newPushKey, 'http://127.0.0.1:3000/admin') .then(result => { expect(result).toBe(true) expect(FetchMock.callCount()).toBe(0) }) }) test('refuses a pushed url pointing at the cloud metadata endpoint', () => { FetchMock.route(/.*/, () => FetchMock.html('should never load')) return Attachment.pushUrl(newPushKey, 'http://169.254.169.254/latest/meta-data/') .then(() => { expect(FetchMock.callCount()).toBe(0) }) }) test('resolves truthy for an unknown pushkey without scraping', () => { FetchMock.route(/.*/, () => FetchMock.html('nope')) return Attachment.pushUrl('not-a-real-pushkey', 'https://example.com/') .then(result => { expect(result).toBe(true) expect(FetchMock.callCount()).toBe(0) }) }) // Previously this could hang the HTTP response indefinitely. test('returns within the budget when the server never responds', () => { const url = 'https://www.solidscribe.com/slow' FetchMock.route(url, () => FetchMock.hang()) ScrapeQueue.configure({ perHostMinSpacingMs: 0, perHostMaxSpacingMs: 0, checkRobots: false, pageTimeoutMs: 80, totalBudgetMs: 300, retries: 0, }) const startedAt = Date.now() return Attachment.pushUrl(newPushKey, url) .then(result => { expect(result).toBe(true) expect(Date.now() - startedAt).toBeLessThan(5000) }) }) }) test('Test Delete Push Key', () => { return Attachment.deletePushKey(newUserId) .then(( results => { // Expect a true bool expect(results).toBe(true) })) }) afterAll(done => { // Close Database const db = require('../../config/database') db.end() done() })